diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index ebc22014b0..8c1acdd879 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -5,9 +5,9 @@ on: paths: - .github/workflows/qemu-image-build.yml - ansible/vars.yml + - ebssurrogate/scripts/* - nix/packages/build-qemu-image/* - qemu.pkr.hcl - - scripts/* workflow_dispatch: permissions: diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 4d91ac23b0..448cfa5471 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -211,6 +211,11 @@ build { destination = "/tmp/chroot-bootstrap-nix.sh" } + provisioner "file" { + source = "ebssurrogate/scripts/cleanup.sh" + destination = "/tmp/cleanup.sh" + } + provisioner "file" { source = "ebssurrogate/files/cloud.cfg" destination = "/tmp/cloud.cfg" @@ -241,11 +246,6 @@ build { destination = "/tmp/ansible-playbook" } - provisioner "file" { - source = "scripts" - destination = "/tmp/ansible-playbook" - } - provisioner "file" { source = "ansible/vars.yml" destination = "/tmp/ansible-playbook/vars.yml" diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 1f6064ef39..ee6ebb0230 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -211,6 +211,11 @@ build { destination = "/tmp/chroot-bootstrap-nix.sh" } + provisioner "file" { + source = "ebssurrogate/scripts/cleanup.sh" + destination = "/tmp/cleanup.sh" + } + provisioner "file" { source = "ebssurrogate/files/cloud.cfg" destination = "/tmp/cloud.cfg" @@ -241,11 +246,6 @@ build { destination = "/tmp/ansible-playbook" } - provisioner "file" { - source = "scripts" - destination = "/tmp/ansible-playbook" - } - provisioner "file" { source = "ansible/vars.yml" destination = "/tmp/ansible-playbook/vars.yml" diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh index f6c2c0ae31..90ee5418de 100755 --- a/ebssurrogate/scripts/chroot-bootstrap-nix.sh +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -8,318 +8,152 @@ set -o errexit set -o pipefail set -o xtrace -export DEBIAN_FRONTEND=noninteractive - -export APT_OPTIONS="-oAPT::Install-Recommends=false \ - -oAPT::Install-Suggests=false \ - -oAcquire::Languages=none" - -# Prevent services from starting during package installation in chroot -# This avoids hangs from cloud-init, dbus, etc. trying to start services -cat >/usr/sbin/policy-rc.d <<'EOF' -#!/bin/sh -exit 101 -EOF -chmod +x /usr/sbin/policy-rc.d - -if [ $(dpkg --print-architecture) = "amd64" ]; then - ARCH="amd64" -else - ARCH="arm64" -fi - -# Get current mirror from sources.list -function get_current_mirror { - if [ "${ARCH}" = "amd64" ]; then - grep -oP 'https?://[^/]+(?=/ubuntu/)' /etc/apt/sources.list | head -1 || echo "" - else - grep -oP 'http://[^/]+(?=/ubuntu-ports/)' /etc/apt/sources.list | head -1 || echo "" - fi -} - -# Switch to a different mirror -function switch_mirror { - local new_mirror="$1" - local sources_file="/etc/apt/sources.list" - - echo "Switching to mirror: ${new_mirror}" - if [ "${ARCH}" = "amd64" ]; then - sed -i "s|http://[^/]*/ubuntu/|http://${new_mirror}/ubuntu/|g" "${sources_file}" - else - sed -i "s|http://[^/]*/ubuntu-ports/|http://${new_mirror}/ubuntu-ports/|g" "${sources_file}" - fi - - # Show what we're using - echo "Current sources.list configuration:" - grep -E '^deb ' "${sources_file}" | head -3 -} - -# Get list of mirrors to try -function get_mirror_list { - local sources_file="/etc/apt/sources.list" - local -a mirrors=() - - # Priority order: - # 1. Country-specific mirror (most reliable) - # 2. Regional CDN (can be inconsistent) - # 3. Global fallback - - if [ "${ARCH}" = "amd64" ]; then - local current_region=$(grep -oP '(?<=http://)[^.]+(?=\.ec2\.archive\.ubuntu\.com)' "${sources_file}" | head -1 || echo "") - - if [ -n "${current_region}" ]; then - mirrors+=("${current_region}.ec2.archive.ubuntu.com") - fi - - mirrors+=("archive.ubuntu.com") - else - local current_region=$(grep -oP '(?<=http://)[^.]+(?=\.clouds\.ports\.ubuntu\.com)' "${sources_file}" | head -1 || echo "") - - # Singapore country mirror for ap-southeast-1 - if [ "${current_region}" = "ap-southeast-1" ]; then - mirrors+=("sg.ports.ubuntu.com") - fi - - if [ -n "${current_region}" ]; then - mirrors+=("${current_region}.clouds.ports.ubuntu.com") - fi - - mirrors+=("ports.ubuntu.com") - fi - - echo "${mirrors[@]}" -} - -# Mirror fallback function for resilient apt-get update -function apt_update_with_fallback { - local sources_file="/etc/apt/sources.list" - local -a mirror_list=($(get_mirror_list)) - local attempt=1 - local max_attempts=${#mirror_list[@]} - - for mirror in "${mirror_list[@]}"; do - echo "=========================================" - echo "Attempting apt-get update with mirror: ${mirror}" - echo "Attempt ${attempt} of ${max_attempts}" - echo "=========================================" - - switch_mirror "${mirror}" - - # Attempt update with timeout (5 minutes) - if timeout 300 apt-get $APT_OPTIONS update 2>&1; then - echo "=========================================" - echo "✓ Successfully updated apt cache using mirror: ${mirror}" - echo "=========================================" - return 0 +function setup_apt { + local aptconf + aptconf=$(mktemp) + cat >"$aptconf" <<-EOF + APT::Install-Recommends "false"; + APT::Install-Suggests "false"; + Acquire::Languages "none"; + EOF + export APT_CONFIG=$aptconf DEBIAN_FRONTEND=noninteractive + + local token region + token=$(curl -s -m 1 -X PUT -H 'x-aws-ec2-metadata-token-ttl-seconds: 10' http://169.254.169.254/latest/api/token || :) + region=$(curl -s -m 1 -H "x-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/placement/region || :) + + mv /etc/apt/sources.list.d/ubuntu.sources{,.bak} || : + + { + if [[ $ARCH == arm64 ]]; then + cat <<-EOF + Types: deb + URIs: http://$region.ec2.ports.ubuntu.com/ubuntu-ports/ http://ports.ubuntu.com/ubuntu-ports/ + Suites: noble noble-updates noble-backports + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + + Types: deb + URIs: http://ports.ubuntu.com/ubuntu-ports/ + Suites: noble-security + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + EOF else - local exit_code=$? - echo "=========================================" - echo "✗ Failed to update using mirror: ${mirror}" - echo "Exit code: ${exit_code}" - echo "=========================================" - - # Clean partial downloads - apt-get clean - rm -rf /var/lib/apt/lists/* - - # Exponential backoff before next attempt - if [ ${attempt} -lt ${max_attempts} ]; then - local sleep_time=$((attempt * 5)) - echo "Waiting ${sleep_time} seconds before trying next mirror..." - sleep ${sleep_time} - fi + cat <<-EOF + Types: deb + URIs: http://$region.ec2.archive.ubuntu.com/ubuntu/ http://archive.ubuntu.com/ubuntu/ + Suites: noble noble-updates noble-backports + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + + Types: deb + URIs: http://security.ubuntu.com/ubuntu/ + Suites: noble-security + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + EOF fi - - attempt=$((attempt + 1)) - done - - echo "=========================================" - echo "ERROR: All mirror tiers failed after ${max_attempts} attempts" - echo "=========================================" - return 1 + } >/etc/apt/sources.list.d/ubuntu.sources } -# Wrapper for apt-get install with mirror fallback on 404 errors -function apt_install_with_fallback { - local -a mirror_list=($(get_mirror_list)) - local attempt=1 - local max_attempts=${#mirror_list[@]} - local original_mirror=$(get_current_mirror) - - for mirror in "${mirror_list[@]}"; do - echo "=========================================" - echo "Attempting apt-get install with mirror: ${mirror}" - echo "Attempt ${attempt} of ${max_attempts}" - echo "=========================================" - - switch_mirror "${mirror}" - - # Re-run apt-get update to get package lists from new mirror - if ! timeout 300 apt-get $APT_OPTIONS update 2>&1; then - echo "Warning: apt-get update failed for mirror ${mirror}, trying next..." - attempt=$((attempt + 1)) - continue - fi - - # Run apt-get install directly (no output capture to avoid buffering/timeout issues) - local exit_code=0 - apt-get "$@" || exit_code=$? - - if [ ${exit_code} -eq 0 ]; then - echo "=========================================" - echo "✓ Successfully installed packages using mirror: ${mirror}" - echo "=========================================" - return 0 - fi - - # On failure, check if it's a mirror issue worth retrying - echo "=========================================" - echo "✗ apt-get failed with exit code: ${exit_code}" - echo "=========================================" - - # Clean apt cache before potential retry - apt-get clean - - if [ ${attempt} -lt ${max_attempts} ]; then - local sleep_time=$((attempt * 5)) - echo "Waiting ${sleep_time} seconds before trying next mirror..." - sleep ${sleep_time} - fi - - attempt=$((attempt + 1)) - done +function cleanup_apt { + if [[ -f /etc/apt/sources.list.d/ubuntu.sources.bak ]]; then + mv /etc/apt/sources.list.d/ubuntu.sources{.bak,} + fi - echo "=========================================" - echo "ERROR: All mirror tiers failed for apt-get install after ${max_attempts} attempts" - echo "=========================================" - return 1 + apt-get clean --yes + apt-get autoremove --yes } -function update_install_packages { - source /etc/os-release +function update_and_upgrade_apt { + apt-get update --yes + apt-get upgrade --yes + apt-get dist-upgrade --yes +} - # Update APT with new sources (using fallback mechanism) - cat /etc/apt/sources.list - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists with any mirror tier" - exit 1 - fi - apt-get $APT_OPTIONS --yes dist-upgrade +function install_initial_packages { + local packages=( + e2fsprogs + initramfs-tools + linux-aws + ) # Do not configure grub during package install - if [ "${ARCH}" = "amd64" ]; then + if [[ $ARCH == arm64 ]]; then + packages+=( + cloud-guest-utils + dosfstools + efibootmgr + fdisk + grub-efi-arm64 + ) + else echo 'grub-pc grub-pc/install_devices_empty select true' | debconf-set-selections echo 'grub-pc grub-pc/install_devices select' | debconf-set-selections # Install various packages needed for a booting system (with mirror fallback) - if ! apt_install_with_fallback install -y linux-aws grub-pc e2fsprogs; then - echo "FATAL: Failed to install boot packages" - exit 1 - fi - else - if ! apt_install_with_fallback install -y e2fsprogs; then - echo "FATAL: Failed to install e2fsprogs" - exit 1 - fi + packages+=(grub-pc) + fi + if ! apt-get install --yes "${packages[@]}"; then + echo "FATAL: Failed to install boot packages" + exit 1 fi + # Install standard packages (with mirror fallback) # Note: ec2-hibinit-agent, ec2-instance-connect, hibagent moved to stage 2 # because their post-install scripts try to access EC2 metadata service # which doesn't work in a chroot and causes long hangs - if ! apt_install_with_fallback install -y \ - sudo \ - wget \ - cloud-init \ - acpid \ - ncurses-term \ - ssh-import-id; then + packages=( + acpid + at + cloud-init + cron + fail2ban + git + less + locales + logrotate + ncurses-term + openssh-server + python3-systemd + ssh-import-id + sudo + ufw + wget + ) + if ! apt-get install --yes "${packages[@]}"; then echo "FATAL: Failed to install standard packages" exit 1 fi - - # apt upgrade - apt-get upgrade -y - - # Install OpenSSH and other packages - sudo add-apt-repository --yes universe - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists after adding universe repository" - exit 1 - fi - if ! apt_install_with_fallback install -y --no-install-recommends \ - openssh-server \ - git \ - ufw \ - cron \ - logrotate \ - fail2ban \ - locales \ - at \ - less \ - python3-systemd; then - echo "FATAL: Failed to install universe packages" - exit 1 - fi - - if [ "${ARCH}" = "arm64" ]; then - if ! apt_install_with_fallback $APT_OPTIONS --yes install linux-aws initramfs-tools dosfstools; then - echo "FATAL: Failed to install arm64 boot packages" - exit 1 - fi - else - if ! apt_install_with_fallback $APT_OPTIONS --yes install initramfs-tools; then - echo "FATAL: Failed to install amd64 boot packages" - exit 1 - fi - fi } function setup_locale { - cat <>/etc/locale.gen -en_US.UTF-8 UTF-8 -EOF - - cat </etc/default/locale -LANG="C.UTF-8" -LC_CTYPE="C.UTF-8" -EOF + cat >>/etc/locale.gen <<-EOF + en_US.UTF-8 UTF-8 + EOF + cat >/etc/default/locale <<-EOF + LANG="C.UTF-8" + LC_CTYPE="C.UTF-8" + EOF locale-gen en_US.UTF-8 } function setup_postgesql_env { # Create the directory if it doesn't exist - sudo mkdir -p /etc/environment.d + mkdir -p /etc/environment.d # Define the contents of the PostgreSQL environment file - cat </dev/null -LOCALE_ARCHIVE=/usr/lib/locale/locale-archive -LANG="en_US.UTF-8" -LANGUAGE="en_US.UTF-8" -LC_ALL="en_US.UTF-8" -LC_CTYPE="en_US.UTF-8" -EOF -} - -function install_packages_for_build { - apt-get install -y --no-install-recommends linux-libc-dev \ - acl \ - magic-wormhole sysstat \ - build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libsystemd-dev libpq-dev libxml2-utils uuid-dev xsltproc ssl-cert \ - gcc-10 g++-10 \ - libgeos-dev libproj-dev libgdal-dev libjson-c-dev libboost-all-dev libcgal-dev libmpfr-dev libgmp-dev cmake \ - libkrb5-dev \ - maven default-jre default-jdk \ - curl gpp apt-transport-https cmake libc++-dev libc++abi-dev libc++1 libglib2.0-dev libtinfo5 libc++abi1 ninja-build python \ - liblzo2-dev - - source /etc/os-release - - apt-get install -y --no-install-recommends llvm-11-dev clang-11 - # Mark llvm as manual to prevent auto removal - apt-mark manual libllvm11:arm64 + tee /etc/environment.d/postgresql.env >/dev/null <<-EOF + LOCALE_ARCHIVE=/usr/lib/locale/locale-archive + LANG="en_US.UTF-8" + LANGUAGE="en_US.UTF-8" + LC_ALL="en_US.UTF-8" + LC_CTYPE="en_US.UTF-8" + EOF } function setup_apparmor { - if ! apt_install_with_fallback install -y apparmor apparmor-utils auditd; then + if ! apt-get install --yes apparmor apparmor-utils auditd; then echo "FATAL: Failed to install apparmor packages" exit 1 fi @@ -328,27 +162,19 @@ function setup_apparmor { cp -rv /tmp/apparmor_profiles/* /etc/apparmor.d/ } -function setup_grub_conf { +function setup_grub { # Note: Unknown kernel parameters (like zswap settings on kernels without zswap support) # are safely ignored by the kernel and passed to user-space. This allows us to # include them here without risking boot failures on older or incompatible kernels. - cat </etc/default/grub -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_TIMEOUT_STYLE="hidden" -GRUB_DISTRIBUTOR="Supabase postgresql" -GRUB_CMDLINE_LINUX_DEFAULT="nomodeset console=tty1 console=ttyS0 ipv6.disable=0 transparent_hugepage=never zswap.enabled=1 zswap.zpool=zsmalloc zswap.compressor=zstd zswap.max_pool_percent=10" -EOF -} - -# Install GRUB -function install_configure_grub { - setup_grub_conf - if [ "${ARCH}" = "arm64" ]; then - if ! apt_install_with_fallback $APT_OPTIONS --yes install cloud-guest-utils fdisk grub-efi-arm64 efibootmgr; then - echo "FATAL: Failed to install grub packages for arm64" - exit 1 - fi + cat >/etc/default/grub <<-EOF + GRUB_DEFAULT=0 + GRUB_TIMEOUT=0 + GRUB_TIMEOUT_STYLE="hidden" + GRUB_DISTRIBUTOR="Supabase postgresql" + GRUB_CMDLINE_LINUX_DEFAULT="nomodeset console=tty1 console=ttyS0 ipv6.disable=0 transparent_hugepage=never zswap.enabled=1 zswap.zpool=zsmalloc zswap.compressor=zstd zswap.max_pool_percent=10" + EOF + + if [[ $ARCH == arm64 ]]; then rm -rf /etc/grub.d/30_os-prober sleep 1 fi @@ -365,35 +191,37 @@ function setup_hostname { # Set the static hostname echo "ubuntu" >/etc/hostname chmod 644 /etc/hostname + # Update netplan configuration to not send hostname - cat </etc/netplan/01-hostname.yaml -network: - version: 2 - ethernets: - eth0: - dhcp4: true - dhcp4-overrides: - send-hostname: false -EOF + cat >/etc/netplan/01-hostname.yaml <<-EOF + network: + version: 2 + ethernets: + eth0: + dhcp4: true + dhcp4-overrides: + send-hostname: false + EOF # Set proper permissions for netplan security chmod 600 /etc/netplan/01-hostname.yaml } # Set options for the default interface function setup_eth0_interface { - cat </etc/netplan/eth0.yaml -network: - version: 2 - ethernets: - eth0: - dhcp4: true -EOF + cat >/etc/netplan/eth0.yaml <<-EOF + network: + version: 2 + ethernets: + eth0: + dhcp4: true + EOF # Set proper permissions for netplan security chmod 600 /etc/netplan/eth0.yaml } function disable_sshd_passwd_auth { - sed -i -E -e 's/^#?\s*PasswordAuthentication\s+(yes|no)\s*$/PasswordAuthentication no/g' \ + sed -i -E \ + -e 's/^#?\s*PasswordAuthentication\s+(yes|no)\s*$/PasswordAuthentication no/g' \ -e 's/^#?\s*ChallengeResponseAuthentication\s+(yes|no)\s*$/ChallengeResponseAuthentication no/g' \ /etc/ssh/sshd_config } @@ -402,23 +230,19 @@ function create_admin_account { groupadd admin } -#Set default target as multi-user function set_default_target { rm -f /etc/systemd/system/default.target ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target } -# Setup ccache -function setup_ccache { - apt-get install ccache -y - mkdir -p /tmp/ccache - export PATH=/usr/lib/ccache:$PATH - echo "PATH=$PATH" >>/etc/environment -} - -# Clear apt caches -function cleanup_cache { - apt-get clean +# Prevent services from starting during package installation in chroot +# This avoids hangs from cloud-init, dbus, etc. trying to start services +function disable_services { + cat >/usr/sbin/policy-rc.d <<-EOF + #!/bin/sh + exit 101 + EOF + chmod +x /usr/sbin/policy-rc.d } # Remove policy-rc.d so services start normally on boot @@ -426,11 +250,16 @@ function enable_services { rm -f /usr/sbin/policy-rc.d } -update_install_packages +ARCH=$(dpkg --print-architecture) +: "${ARCH:?Failed to detect architecture}" + +disable_services +setup_apt +update_and_upgrade_apt +install_initial_packages setup_locale setup_postgesql_env -#install_packages_for_build -install_configure_grub +setup_grub setup_apparmor setup_hostname create_admin_account @@ -438,6 +267,5 @@ set_default_target setup_eth0_interface disable_sshd_passwd_auth disable_fsck -#setup_ccache -cleanup_cache +cleanup_apt enable_services diff --git a/ebssurrogate/scripts/cleanup-qemu.sh b/ebssurrogate/scripts/cleanup-qemu.sh new file mode 100755 index 0000000000..7179e0dad7 --- /dev/null +++ b/ebssurrogate/scripts/cleanup-qemu.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +# DigitalOcean Marketplace Image Validation Tool +# © 2021 DigitalOcean LLC. +# This code is licensed under Apache 2.0 license (see LICENSE.md for details) + +set -ex + +# Ensure /tmp exists and has the proper permissions before +# checking for security updates +# https://github.com/digitalocean/marketplace-partners/issues/94 +if [[ ! -d /tmp ]]; then + mkdir /tmp +fi +chmod 1777 /tmp + +# Cleanup more packages +packages=( + ansible + ansible-core + apport + appstream + autoconf + automake + autotools-dev + bash-completion + bcache-tools + bind9-dnsutils + bind9-host + bind9-libs + bolt + btrfs-progs + byobu + cmake-data + command-not-found + console-setup + cpp-10 + cpp-9 + distro-info + eject + fonts-dejavu* + fonts-ubuntu-console + friendly-recovery + ftp + fwupd + gawk + gcc-10 + gcc-9 + gdisk + git + git-man + gnupg + keyboard-configuration + libc6-dev + libcgal-dev + libclang-cpp18 + libevent-dev + libgcc-9-dev + libicu-dev + libicu74 + libpcre3-dev + libssl-dev + libsystemd-dev + libvolume-key1 + linux-headers* + linux-libc-dev + lvm2 + lxd-agent-loader + lxd-installer + man-db + mdadm + modemmanager + mtd-utils + multipath-tools + nano + netcat-openbsd + nfs-common + ntfs-3g + open-vm-tools + packagekit + parted + pastebinit + plymouth + publicsuffix + python-babel-localedata + python3-botocore + python3-pygments + python3-twisted + screen + snapd + strace + thin-provisioning-tools + tmux + unattended-upgrades + usb-modeswitch + vim + vim-runtime + wget + whiptail + xauth + xfsprogs +) + +if [[ $(uname -m) == aarch64 ]]; then + packages+=(libgcc-8-dev) +fi + +apt-mark manual libevent-2.1-7t64 +apt-get --yes remove --purge "${packages[@]}" +apt-get --yes autoremove +apt-get --yes autoclean +apt-get --yes update +apt-get --yes upgrade + +systemctl set-default multi-user.target +systemctl disable getty@tty1.service +systemctl mask getty@tty1.service +systemctl mask graphical.target + +rm -rf /tmp/* /var/tmp/* +history -c +cat /dev/null >/root/.bash_history +unset HISTFILE + +journalctl --rotate +journalctl --vacuum-time=1s +find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; +rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +rm -rf /var/lib/cloud/instances/* +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* +touch /etc/ssh/revoked_keys +chmod 600 /etc/ssh/revoked_keys + +cat /dev/null >/var/log/lastlog +cat /dev/null >/var/log/wtmp + +dd if=/dev/zero of=/zerofile & +PID=$! +while [ -d /proc/$PID ]; do + printf "." + sleep 5 +done +sync +rm /zerofile +sync + +fstrim / diff --git a/scripts/90-cleanup.sh b/ebssurrogate/scripts/cleanup.sh old mode 100644 new mode 100755 similarity index 54% rename from scripts/90-cleanup.sh rename to ebssurrogate/scripts/cleanup.sh index d221a9f275..307244bae0 --- a/scripts/90-cleanup.sh +++ b/ebssurrogate/scripts/cleanup.sh @@ -14,37 +14,31 @@ if [[ ! -d /tmp ]]; then fi chmod 1777 /tmp -if [ -n "$(command -v yum)" ]; then - yum update -y - yum clean all -elif [ -n "$(command -v apt-get)" ]; then - # Cleanup more packages - apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ - ansible +# Cleanup more packages +packages=( + ansible + ansible + autoconf + automake + autotools-dev + cmake-data + cpp-10 + cpp-9 + gcc-10 + gcc-9 + git + git-man + libcgal-dev + libgcc-9-dev + libicu-dev +) - # add-apt-repository --yes --remove ppa:ansible/ansible +apt-get --yes remove --purge "${packages[@]}" +apt-get --yes autoremove +apt-get --yes autoclean +apt-get --yes update +apt-get --yes upgrade - source /etc/os-release - - apt-get -y update - apt-get -y upgrade - apt-get -y autoremove - apt-get -y autoclean -fi rm -rf /tmp/* /var/tmp/* history -c cat /dev/null >/root/.bash_history @@ -59,12 +53,14 @@ chmod 600 /etc/ssh/revoked_keys # Securely erase the unused portion of the filesystem GREEN='\033[0;32m' NC='\033[0m' -printf "\n${GREEN}Writing zeros to the remaining disk space to securely -erase the unused portion of the file system. -Depending on your disk size this may take several minutes. -The secure erase will complete successfully when you see:${NC} - dd: writing to '/zerofile': No space left on device\n -Beginning secure erase now\n" +cat <<-EOF + $GREEN + Writing zeros to the remaining disk space to securely erase the unused portion of the file system. + Depending on your disk size this may take several minutes. + The secure erase will complete successfully when you see:$NC + dd: writing to '/zerofile': No space left on device + Beginning secure erase now +EOF dd if=/dev/zero of=/zerofile & PID=$! diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh new file mode 100755 index 0000000000..0b93176b4f --- /dev/null +++ b/ebssurrogate/scripts/nix-provision.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -o errexit +set -o pipefail +set -o xtrace + +exec 1>&2 + +function setup_apt { + local aptconf + aptconf=$(mktemp) + cat >"$aptconf" <<-EOF + APT::Install-Recommends "false"; + APT::Install-Suggests "false"; + Acquire::Languages "none"; + EOF + export APT_CONFIG=$aptconf DEBIAN_FRONTEND=noninteractive +} + +function cleanup_apt { + apt-get clean --yes + apt-get autoremove --yes +} + +function update_and_upgrade_apt { + apt-get update --yes + apt-get upgrade --yes +} + +function install_packages { + # Install EC2-specific packages that were deferred from stage 1 + # These packages have post-install scripts that need EC2 metadata service access + # which only works on a real running EC2 instance (not in chroot) + packages=( + ec2-hibinit-agent + ec2-instance-connect + hibagent + ) + + # Setup Ansible on host VM + # apt-get update && apt-get install -y software-properties-common + # + # Manually add GPG key with explicit keyserver + # apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 93C4A3FD7BB9C367 + # + # Add repository and install + # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out + # add-apt-repository --yes ppa:ansible/ansible + # apt-get update + packages+=(ansible) + + apt-get install --yes "${packages[@]}" + ansible-galaxy collection install community.general +} + +function install_nix() { + curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --yes --daemon --nix-extra-conf-file <( + cat <<-EOF + extra-experimental-features = nix-command flakes + extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com + extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= + EOF + ) + + #shellcheck disable=SC1091 + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + nix --version +} + +function execute_stage2_playbook { + echo "POSTGRES_MAJOR_VERSION: $POSTGRES_MAJOR_VERSION" + echo "GIT_SHA: $GIT_SHA" + mkdir -p /etc/ansible + tee /etc/ansible/ansible.cfg <<-EOF + [defaults] + callbacks_enabled = timer, profile_tasks, profile_roles + EOF + sed -i 's/- hosts: all/- hosts: localhost/' /tmp/ansible-playbook/ansible/playbook.yml + + # Run Ansible playbook + export ANSIBLE_LOG_PATH=/tmp/ansible.log + export ANSIBLE_REMOTE_TEMP=/tmp + + # shellcheck disable=SC2086 + ansible-playbook /tmp/ansible-playbook/ansible/playbook.yml \ + --extra-vars '{"stage2":true, "qemu":false}' \ + --extra-vars "git_commit_sha=$GIT_SHA" \ + --extra-vars "psql_version=psql_$POSTGRES_MAJOR_VERSION" \ + --extra-vars "postgresql_version=postgresql_$POSTGRES_MAJOR_VERSION" \ + --extra-vars "nix_secret_key=$NIX_SECRET_KEY" \ + --extra-vars "postgresql_major_version=$POSTGRES_MAJOR_VERSION" \ + $ARGS +} + +function cleanup_packages { + # add-apt-repository --yes --remove ppa:ansible/ansible + apt-get --y remove --purge ansible +} + +setup_apt +update_and_upgrade_apt +install_packages +install_nix +execute_stage2_playbook +cleanup_packages +update_and_upgrade_apt +cleanup_apt diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 0c432cb696..5b17c0da12 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -4,16 +4,83 @@ set -o errexit set -o pipefail set -o xtrace -if [ $(dpkg --print-architecture) = "amd64" ]; then - ARCH="amd64" -else - ARCH="arm64" -fi +################# +# stage1 things # +################# -function waitfor_boot_finished { - export DEBIAN_FRONTEND=noninteractive +function setup_apt { + local aptconf + aptconf=$(mktemp) + cat >"$aptconf" <<-EOF + APT::Install-Recommends "false"; + APT::Install-Suggests "false"; + Acquire::Languages "none"; + EOF + export APT_CONFIG=$aptconf DEBIAN_FRONTEND=noninteractive + + # Qemu is not like the others because we can run it locally :toot:! + local token + token=$(curl -s -m 1 -X PUT -H 'x-aws-ec2-metadata-token-ttl-seconds: 10' http://169.254.169.254/latest/api/token || :) + if [[ -z $token ]]; then + return + fi + + local region + region=$(curl -s -m 1 -H "x-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/placement/region || :) + if [[ -z $region ]]; then + return + fi + + mv /etc/apt/sources.list.d/ubuntu.sources{,.bak} || : + + { + if [[ $ARCH == arm64 ]]; then + cat <<-EOF + Types: deb + URIs: http://$region.ec2.ports.ubuntu.com/ubuntu-ports/ http://ports.ubuntu.com/ubuntu-ports/ + Suites: noble noble-updates noble-backports + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg - echo "args: ${ARGS}" + Types: deb + URIs: http://ports.ubuntu.com/ubuntu-ports/ + Suites: noble-security + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + EOF + else + cat <<-EOF + Types: deb + URIs: http://$region.ec2.archive.ubuntu.com/ubuntu/ http://archive.ubuntu.com/ubuntu/ + Suites: noble noble-updates noble-backports + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + + Types: deb + URIs: http://security.ubuntu.com/ubuntu/ + Suites: noble-security + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + EOF + fi + } >/etc/apt/sources.list.d/ubuntu.sources +} + +function cleanup_apt { + if [[ -f /etc/apt/sources.list.d/ubuntu.sources.bak ]]; then + mv /etc/apt/sources.list.d/ubuntu.sources{.bak,} + fi + + apt-get clean --yes + apt-get autoremove --yes +} + +function update_and_upgrade_apt { + apt-get update --yes + apt-get upgrade --yes +} + +function waitfor_boot_finished { # Wait for cloudinit on the surrogate to complete before making progress while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do echo 'Waiting for cloud-init...' @@ -22,137 +89,142 @@ function waitfor_boot_finished { } function install_packages { - apt-get update - apt-get install -y \ - arptables \ - e2fsprogs \ - ebtables \ - gpg \ - iptables \ - less \ - locales \ - logrotate \ - nfs-common \ - software-properties-common \ - ufw \ - ; + packages=( + ansible + arptables + e2fsprogs + ebtables + gpg + iptables + less + locales + logrotate + nfs-common + software-properties-common + ufw + ) + apt-get install --yes "${packages[@]}" + # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out # add-apt-repository --yes --update ppa:ansible/ansible && - sudo apt-get install ansible -y ansible-galaxy collection install community.general } function execute_playbook { - sudo mkdir -p /etc/ansible - tee /etc/ansible/ansible.cfg </dev/null -LOCALE_ARCHIVE=/usr/lib/locale/locale-archive -LANG="en_US.UTF-8" -LANGUAGE="en_US.UTF-8" -LC_ALL="en_US.UTF-8" -LC_CTYPE="en_US.UTF-8" -EOF + tee /etc/environment.d/postgresql.env >/dev/null <<-EOF + LOCALE_ARCHIVE=/usr/lib/locale/locale-archive + LANG="en_US.UTF-8" + LANGUAGE="en_US.UTF-8" + LC_ALL="en_US.UTF-8" + LC_CTYPE="en_US.UTF-8" + EOF } function setup_locale { - cat <>/etc/locale.gen -en_US.UTF-8 UTF-8 -EOF - - cat </etc/default/locale -LANG="C.UTF-8" -LC_CTYPE="C.UTF-8" -EOF + cat >>/etc/locale.gen <<-EOF + en_US.UTF-8 UTF-8 + EOF + + cat >/etc/default/locale <<-EOF + LANG="C.UTF-8" + LC_CTYPE="C.UTF-8" + EOF locale-gen en_US.UTF-8 } -sed -i 's/- hosts: all/- hosts: localhost/' ansible/playbook.yml - -waitfor_boot_finished -install_packages -setup_postgesql_env -setup_locale -execute_playbook - -#################### -# stage 2 things -#################### +################# +# stage2 things # +################# function install_nix() { - sudo su -c "sh <(curl -L https://releases.nixos.org/nix/nix-2.34.6/install) --yes --daemon --nix-extra-conf-file /dev/stdin <&2 -if [ $(dpkg --print-architecture) = "amd64" ]; then - ARCH="amd64" -else - ARCH="arm64" -fi - -# Mirror fallback function for resilient apt-get update -function apt_update_with_fallback { - local sources_file="/etc/apt/sources.list" - local max_attempts=2 - local attempt=1 - - # Get EC2 region if not already set - if [ -z "${REGION}" ]; then - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g' || echo "") - fi - - # Define mirror tiers (in priority order) - local -a mirror_tiers=() - if [ "${ARCH}" = "amd64" ]; then - if [ -n "${REGION}" ]; then - mirror_tiers+=("${REGION}.ec2.archive.ubuntu.com") - fi - mirror_tiers+=("archive.ubuntu.com") - else - if [ -n "${REGION}" ]; then - mirror_tiers+=("${REGION}.clouds.ports.ubuntu.com") - fi - mirror_tiers+=("ports.ubuntu.com") - fi +function setup_apt { + local aptconf + aptconf=$(mktemp) + cat >"$aptconf" <<-EOF + APT::Install-Recommends "false"; + APT::Install-Suggests "false"; + Acquire::Languages "none"; + EOF + export APT_CONFIG=$aptconf DEBIAN_FRONTEND=noninteractive + + local token region + token=$(curl -s -m 1 -X PUT -H 'x-aws-ec2-metadata-token-ttl-seconds: 10' http://169.254.169.254/latest/api/token || :) + region=$(curl -s -m 1 -H "x-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/placement/region || :) + + mv /etc/apt/sources.list.d/ubuntu.sources{,.bak} || : - # If we couldn't get REGION, skip tier 1 - if [ -z "${REGION}" ]; then - echo "Warning: Could not determine EC2 region, skipping regional mirror" - mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element - fi - - for mirror in "${mirror_tiers[@]}"; do - echo "=========================================" - echo "Attempting apt-get update with mirror: ${mirror}" - echo "Attempt ${attempt} of ${max_attempts}" - echo "=========================================" - - # Update sources.list to use current mirror - if [ "${ARCH}" = "amd64" ]; then - sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}" + { + if [[ $ARCH == arm64 ]]; then + cat <<-EOF + Types: deb + URIs: http://$region.ec2.ports.ubuntu.com/ubuntu-ports/ http://ports.ubuntu.com/ubuntu-ports/ + Suites: noble noble-updates noble-backports + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + + Types: deb + URIs: http://ports.ubuntu.com/ubuntu-ports/ + Suites: noble-security + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + EOF else - sed -i "s|http://[^/]*/ubuntu-ports/|http://${mirror}/ubuntu-ports/|g" "${sources_file}" - sed -i "s|http://ports.ubuntu.com/ubuntu-ports|http://${mirror}/ubuntu-ports|g" "${sources_file}" + cat <<-EOF + Types: deb + URIs: http://$region.ec2.archive.ubuntu.com/ubuntu/ http://archive.ubuntu.com/ubuntu/ + Suites: noble noble-updates noble-backports + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + + Types: deb + URIs: http://security.ubuntu.com/ubuntu/ + Suites: noble-security + Components: main restricted universe multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + EOF fi + } >/etc/apt/sources.list.d/ubuntu.sources +} - # Show what we're using - echo "Current sources.list configuration:" - grep -E '^deb ' "${sources_file}" | head -3 - - # Attempt update with timeout (5 minutes) - if timeout 300 apt-get update 2>&1; then - echo "=========================================" - echo "✓ Successfully updated apt cache using mirror: ${mirror}" - echo "=========================================" - return 0 - else - local exit_code=$? - echo "=========================================" - echo "✗ Failed to update using mirror: ${mirror}" - echo "Exit code: ${exit_code}" - echo "=========================================" - - # Clean partial downloads - apt-get clean - rm -rf /var/lib/apt/lists/* - - # Exponential backoff before next attempt - if [ ${attempt} -lt ${max_attempts} ]; then - local sleep_time=$((attempt * 5)) - echo "Waiting ${sleep_time} seconds before trying next mirror..." - sleep ${sleep_time} - fi - fi +function cleanup_apt { + if [[ -f /etc/apt/sources.list.d/ubuntu.sources.bak ]]; then + mv /etc/apt/sources.list.d/ubuntu.sources{.bak,} + fi - attempt=$((attempt + 1)) - done + apt-get clean --yes + apt-get autoremove --yes +} - echo "=========================================" - echo "ERROR: All mirror tiers failed after ${max_attempts} attempts" - echo "=========================================" - return 1 +function update_and_upgrade_apt { + apt-get update --yes + apt-get upgrade --yes } function waitfor_boot_finished { - export DEBIAN_FRONTEND=noninteractive - - echo "args: ${ARGS}" - # Wait for cloudinit on the surrogate to complete before making progress while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do echo 'Waiting for cloud-init...' sleep 1 @@ -113,35 +83,25 @@ function waitfor_boot_finished { } function install_packages { - # Setup Ansible on host VM - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists on host VM" - exit 1 - fi + packages=( + ansible + debootstrap + e2fsprogs + gdisk + nvme-cli + ) + apt-get install --yes "${packages[@]}" - sudo apt-get install software-properties-common -y # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out + # apt-get install --yes software-properties-common # add-apt-repository --yes --update ppa:ansible/ansible - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists after adding Ansible PPA" - exit 1 - fi - - sudo apt-get install ansible -y ansible-galaxy collection install community.general - - apt-get install -y \ - gdisk \ - e2fsprogs \ - debootstrap \ - nvme-cli } # Partition the new root EBS volume function create_partition_table { - - if [ "${ARCH}" = "arm64" ]; then + if [[ $ARCH == arm64 ]]; then parted --script /dev/xvdf \ mklabel gpt \ mkpart UEFI 1MiB 100MiB \ @@ -159,46 +119,43 @@ function create_partition_table { function device_partition_mappings { # NVMe EBS launch device mappings (symlinks): /dev/nvme*n* to /dev/xvd* declare -A blkdev_mappings - for blkdev in $( # /dev/nvme*n* - nvme list | awk '/^\/dev/ { print $1 }' - ); do + while read -r blkdev; do # Mapping info from disk headers - header=$(nvme id-ctrl --raw-binary "${blkdev}" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g' | sed 's!/dev/!!') - mapping="/dev/${header%%[0-9]}" # normalize sda1 => sda + header=$(nvme id-ctrl --raw-binary "$blkdev" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g' | sed 's!/dev/!!') + mapping=/dev/${header%%[0-9]} # normalize sda1 => sda # Create /dev/xvd* device symlink - if [[ -n $mapping ]] && [[ -b ${blkdev} ]] && [[ ! -L ${mapping} ]]; then + if [[ -n $mapping ]] && [[ -b $blkdev ]] && [[ ! -L $mapping ]]; then ln -s "$blkdev" "$mapping" - blkdev_mappings["$blkdev"]="$mapping" + blkdev_mappings[$blkdev]=$mapping fi - done + done < <(nvme list | awk '/^\/dev/ { print $1 }') create_partition_table # NVMe EBS launch device partition mappings (symlinks): /dev/nvme*n*p* to /dev/xvd*[0-9]+ declare -A partdev_mappings for blkdev in "${!blkdev_mappings[@]}"; do # /dev/nvme*n* - mapping="${blkdev_mappings[$blkdev]}" + mapping=${blkdev_mappings[$blkdev]} # Create /dev/xvd*[0-9]+ partition device symlink - for partdev in "${blkdev}"p*; do + for partdev in "$blkdev"p*; do partnum=${partdev##*p} - if [[ ! -L "${mapping}${partnum}" ]]; then - ln -s "${blkdev}p${partnum}" "${mapping}${partnum}" - - partdev_mappings["${blkdev}p${partnum}"]="${mapping}${partnum}" + if [[ ! -L "$mapping$partnum" ]]; then + ln -s "${blkdev}p$partnum" "$mapping$partnum" + partdev_mappings[${blkdev}p$partnum]=$mapping$partnum fi done done } -#Download and install latest e2fsprogs for fast_commit feature,if required. +# Download and install latest e2fsprogs for fast_commit feature,if required. function format_and_mount_rootfs { mkfs.ext4 -m0.1 /dev/xvdf2 mount -o noatime,nodiratime /dev/xvdf2 /mnt - if [ "${ARCH}" = "arm64" ]; then + if [[ $ARCH == arm64 ]]; then mkfs.fat -F32 /dev/xvdf1 mkdir -p /mnt/boot/efi sleep 2 @@ -210,8 +167,7 @@ function format_and_mount_rootfs { # Explicitly reserving 100MiB worth of blocks for the data volume # # Any changes here should be propagated to $GIT_DATA_DIR/ansible/files/admin_api_scripts/grow_fs.sh - RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) - tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh + tune2fs -r $((100 * 1024 * 1024 / 4096)) /dev/xvdh mkdir -p /mnt/data mount -o defaults,discard /dev/xvdh /mnt/data @@ -226,58 +182,52 @@ function create_swapfile { function format_build_partition { mkfs.ext4 -O ^has_journal /dev/xvdc } -function pull_docker { - apt-get install -y docker.io - docker run -itd --name ccachedata "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" sh - docker exec -itd ccachedata mkdir -p /build/ccache -} - # Create fstab function create_fstab { - FMT="%-42s %-11s %-5s %-17s %-5s %s" - local ROOT_LINE=$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') - local DATA_LINE=$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') - local SWAP_LINE=$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") + local FMT="%-42s %-11s %-5s %-17s %-5s %s" ROOT_LINE DATA_LINE SWAP_LINE + ROOT_LINE=$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') + DATA_LINE=$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') + + # shellcheck disable=SC2059 + SWAP_LINE=$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") local EFI_LINE="" - if [ "${ARCH}" = "arm64" ]; then - EFI_LINE=$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') + if [[ $ARCH == arm64 ]]; then + EFI_LINE=$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') fi { - printf "${FMT}\n" "# DEVICE UUID" "MOUNTPOINT" "TYPE" "OPTIONS" "DUMP" "FSCK" - echo "${ROOT_LINE}" - [ -n "${EFI_LINE}" ] && echo "${EFI_LINE}" - echo "${DATA_LINE}" - echo "${SWAP_LINE}" - } >"/mnt/etc/fstab" - unset FMT + # shellcheck disable=SC2059 + printf "$FMT\n" "# DEVICE UUID" "MOUNTPOINT" "TYPE" "OPTIONS" "DUMP" "FSCK" + echo "$ROOT_LINE" + [ -n "$EFI_LINE" ] && echo "$EFI_LINE" + echo "$DATA_LINE" + echo "$SWAP_LINE" + } >/mnt/etc/fstab } function setup_chroot_environment { + local UBUNTU_VERSION UBUNTU_VERSION=$(lsb_release -cs) # 'noble' for Ubuntu 24.04 # sometimes debootstrap will get stuck on a download for a long time # the default read timeout in wget is 900s, which can cause a ~15min increase in build time # this forces the process to fail-fast and retry - cat <~/.wgetrc -read_timeout = 30 -timeout = 35 -tries = 5 -EOF - - # Update ec2-region - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g') - - # Bootstrap Ubuntu into /mnt using the regional mirror (avoids global mirror stalls) - if [ "${ARCH}" = "amd64" ]; then - debootstrap --arch ${ARCH} --variant=minbase "$UBUNTU_VERSION" /mnt "http://${REGION}.ec2.archive.ubuntu.com/ubuntu" - else - debootstrap --arch ${ARCH} --variant=minbase "$UBUNTU_VERSION" /mnt "http://${REGION}.clouds.ports.ubuntu.com/ubuntu-ports" - fi - - sed -i "s/REGION/${REGION}/g" /tmp/sources.list - cp /tmp/sources.list /mnt/etc/apt/sources.list + cat >~/.wgetrc <<-EOF + read_timeout = 30 + timeout = 35 + tries = 5 + EOF + + local mirror + # Use the regional mirror we setup for this run, which is the first URI/preferred + mirror=$(awk '/URIs/ {print $2}' /etc/apt/sources.list.d/ubuntu.sources | head -n1) + debootstrap --arch "$ARCH" --variant=minbase "$UBUNTU_VERSION" /mnt "$mirror" + + local region + # parse the region from regional mirror uri + region=$(awk -F'[/.]' '{print $3}' <<<"$mirror") + sed "s/REGION/$region/g" /tmp/sources.list >/mnt/etc/apt/sources.list create_fstab @@ -303,10 +253,10 @@ EOF cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh chroot /mnt /tmp/chroot-bootstrap-nix.sh rm -f /mnt/tmp/chroot-bootstrap-nix.sh - echo "${POSTGRES_SUPABASE_VERSION}" >/mnt/root/supabase-release + echo "$POSTGRES_SUPABASE_VERSION" >/mnt/root/supabase-release # Copy the AMI version into the /etc/supabase-release file - echo "${POSTGRES_SUPABASE_VERSION}" >/mnt/etc/supabase-release + echo "$POSTGRES_SUPABASE_VERSION" >/mnt/etc/supabase-release chmod 644 /mnt/etc/supabase-release # Copy the nvme identification script into /sbin inside the chroot @@ -326,23 +276,23 @@ EOF sleep 2 } -function download_ccache { - docker cp ccachedata:/build/ccache/. /mnt/tmp/ccache -} - function execute_playbook { - sudo mkdir -p /etc/ansible - tee /etc/ansible/ansible.cfg <>/etc/ssh/sshd_config </root/.bash_history -unset HISTFILE - -journalctl --rotate -journalctl --vacuum-time=1s -find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; -rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? -rm -rf /var/lib/cloud/instances/* -rm -f /root/.ssh/authorized_keys /etc/ssh/*key* -touch /etc/ssh/revoked_keys -chmod 600 /etc/ssh/revoked_keys - -cat /dev/null >/var/log/lastlog -cat /dev/null >/var/log/wtmp - -dd if=/dev/zero of=/zerofile & -PID=$! -while [ -d /proc/$PID ]; do - printf "." - sleep 5 -done -sync -rm /zerofile -sync - -fstrim / diff --git a/scripts/91-log_cleanup.sh b/scripts/91-log_cleanup.sh deleted file mode 100644 index 648c46d354..0000000000 --- a/scripts/91-log_cleanup.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -#Erasing all logs -# -echo "Clearing all log files" -rm -rf /var/log/* - -# creating system stats directory -mkdir /var/log/sysstat - -# https://github.com/fail2ban/fail2ban/issues/1593 -touch /var/log/auth.log - -touch /var/log/pgbouncer.log -chown pgbouncer:postgres /var/log/pgbouncer.log - -mkdir /var/log/postgresql -chown postgres:postgres /var/log/postgresql - -mkdir /var/log/wal-g -cd /var/log/wal-g -touch backup-push.log backup-fetch.log wal-push.log wal-fetch.log pitr.log -chown -R postgres:postgres /var/log/wal-g -chmod -R 0300 /var/log/wal-g diff --git a/scripts/99-img_check.sh b/scripts/99-img_check.sh deleted file mode 100755 index 205ad07f81..0000000000 --- a/scripts/99-img_check.sh +++ /dev/null @@ -1,669 +0,0 @@ -#!/bin/bash - -# DigitalOcean Marketplace Image Validation Tool -# © 2021 DigitalOcean LLC. -# This code is licensed under Apache 2.0 license (see LICENSE.md for details) - -VERSION="v. 1.6" -RUNDATE=$(date) - -# Script should be run with SUDO -if [ "$EUID" -ne 0 ]; then - echo "[Error] - This script must be run with sudo or as the root user." - exit 1 -fi - -STATUS=0 -PASS=0 -WARN=0 -FAIL=0 - -# $1 == command to check for -# returns: 0 == true, 1 == false -cmdExists() { - if command -v "$1" >/dev/null 2>&1; then - return 0 - else - return 1 - fi -} - -function getDistro { - if [ -f /etc/os-release ]; then - # freedesktop.org and systemd - . /etc/os-release - OS=$NAME - VER=$VERSION_ID - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - OS=$(lsb_release -si) - VER=$(lsb_release -sr) - elif [ -f /etc/lsb-release ]; then - # For some versions of Debian/Ubuntu without lsb_release command - . /etc/lsb-release - OS=$DISTRIB_ID - VER=$DISTRIB_RELEASE - elif [ -f /etc/debian_version ]; then - # Older Debian/Ubuntu/etc. - OS=Debian - VER=$(cat /etc/debian_version) - elif [ -f /etc/SuSe-release ]; then - # Older SuSE/etc. - : - elif [ -f /etc/redhat-release ]; then - # Older Red Hat, CentOS, etc. - VER=$(cat /etc/redhat-release | cut -d" " -f3 | cut -d "." -f1) - d=$(cat /etc/redhat-release | cut -d" " -f1 | cut -d "." -f1) - if [[ $d == "CentOS" ]]; then - OS="CentOS Linux" - fi - else - # Fall back to uname, e.g. "Linux ", also works for BSD, etc. - OS=$(uname -s) - VER=$(uname -r) - fi -} -function loadPasswords { - SHADOW=$(cat /etc/shadow) -} - -function checkAgent { - # Check for the presence of the do-agent in the filesystem - if [ -d /var/opt/digitalocean/do-agent ]; then - echo -en "\e[41m[FAIL]\e[0m DigitalOcean Monitoring Agent detected.\n" - ((FAIL++)) - STATUS=2 - if [[ $OS == "CentOS Linux" ]]; then - echo "The agent can be removed with 'sudo yum remove do-agent' " - elif [[ $OS == "Ubuntu" ]]; then - echo "The agent can be removed with 'sudo apt-get purge do-agent' " - fi - else - echo -en "\e[32m[PASS]\e[0m DigitalOcean Monitoring agent was not found\n" - ((PASS++)) - fi -} - -function checkLogs { - cp_ignore="/var/log/cpanel-install.log" - echo -en "\nChecking for log files in /var/log\n\n" - # Check if there are log archives or log files that have not been recently cleared. - for f in /var/log/*-????????; do - [[ -e $f ]] || break - if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m Log archive ${f} found\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - done - for f in /var/log/*.[0-9]; do - [[ -e $f ]] || break - echo -en "\e[93m[WARN]\e[0m Log archive ${f} found\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - done - for f in /var/log/*.log; do - [[ -e $f ]] || break - if [[ ${f} == '/var/log/lfd.log' && "$(cat "${f}" | egrep -v '/var/log/messages has been reset| Watching /var/log/messages' | wc -c)" -gt 50 ]]; then - if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - elif [[ ${f} != '/var/log/lfd.log' && "$(cat "${f}" | wc -c)" -gt 50 ]]; then - if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - fi - done -} -function checkTMP { - # Check the /tmp directory to ensure it is empty. Warn on any files found. - return 1 -} -function checkRoot { - user="root" - uhome="/root" - for usr in $SHADOW; do - IFS=':' read -r -a u <<<"$usr" - if [[ ${u[0]} == "${user}" ]]; then - if [[ ${u[1]} == "!" ]] || [[ ${u[1]} == "!!" ]] || [[ ${u[1]} == "*" ]]; then - echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" - ((PASS++)) - else - echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account.\n" - ((FAIL++)) - STATUS=2 - fi - fi - done - if [ -d ${uhome}/ ]; then - if [ -d ${uhome}/.ssh/ ]; then - if ls ${uhome}/.ssh/* >/dev/null 2>&1; then - for key in ${uhome}/.ssh/*; do - if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then - - if [ "$(cat "${key}" | wc -c)" -gt 50 ]; then - echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) - echo "File Contents:" - echo $akey - echo "--------------" - ((FAIL++)) - STATUS=2 - fi - elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then - if [ "$(cat "${key}" | wc -c)" -gt 0 ]; then - echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) - echo "File Contents:" - echo $akey - echo "--------------" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - elif [ "${key}" != "${uhome}/.ssh/known_hosts" ]; then - echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a file in their .ssh directory at \e[93m${key}\e[0m\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - else - if [ "$(cat "${key}" | wc -c)" -gt 50 ]; then - echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a populated known_hosts file in \e[93m${key}\e[0m\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - fi - done - else - echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m has no SSH keys present\n" - fi - else - echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have an .ssh directory\n" - fi - if [ -f /root/.bash_history ]; then - - BH_S=$(cat /root/.bash_history | wc -c) - - if [[ $BH_S -lt 200 ]]; then - echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" - ((PASS++)) - else - echo -en "\e[41m[FAIL]\e[0m ${user}'s Bash History should be cleared to prevent sensitive information from leaking\n" - ((FAIL++)) - STATUS=2 - fi - - return 1 - else - echo -en "\e[32m[PASS]\e[0m The Root User's Bash History is not present\n" - ((PASS++)) - fi - else - echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have a directory in /home\n" - fi - echo -en "\n\n" - return 1 -} - -function checkUsers { - # Check each user-created account - for user in $(awk -F: '$3 >= 1000 && $1 != "nobody" {print $1}' /etc/passwd); do - # Skip some other non-user system accounts - if [[ $user == "centos" ]]; then - : - elif [[ $user == "nfsnobody" ]]; then - : - else - echo -en "\nChecking user: ${user}...\n" - for usr in $SHADOW; do - IFS=':' read -r -a u <<<"$usr" - if [[ ${u[0]} == "${user}" ]]; then - if [[ ${u[1]} == "!" ]] || [[ ${u[1]} == "!!" ]] || [[ ${u[1]} == "*" ]]; then - echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" - ((PASS++)) - else - echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account. Only system users are allowed on the image.\n" - ((FAIL++)) - STATUS=2 - fi - fi - done - #echo "User Found: ${user}" - uhome="/home/${user}" - if [ -d "${uhome}/" ]; then - if [ -d "${uhome}/.ssh/" ]; then - if ls "${uhome}/.ssh/*" >/dev/null 2>&1; then - for key in ${uhome}/.ssh/*; do - if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then - if [ "$(cat "${key}" | wc -c)" -gt 50 ]; then - echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) - echo "File Contents:" - echo $akey - echo "--------------" - ((FAIL++)) - STATUS=2 - fi - elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then - if [ "$(cat "${key}" | wc -c)" -gt 0 ]; then - echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) - echo "File Contents:" - echo $akey - echo "--------------" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - elif [ "${key}" != "${uhome}/.ssh/known_hosts" ]; then - - echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a file in their .ssh directory named \e[93m${key}\e[0m\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - - else - if [ "$(cat "${key}" | wc -c)" -gt 50 ]; then - echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a known_hosts file in \e[93m${key}\e[0m\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - fi - - done - else - echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m has no SSH keys present\n" - fi - else - echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have an .ssh directory\n" - fi - else - echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have a directory in /home\n" - fi - - # Check for an uncleared .bash_history for this user - if [ -f "${uhome}/.bash_history" ]; then - BH_S=$(cat "${uhome}/.bash_history" | wc -c) - - if [[ $BH_S -lt 200 ]]; then - echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" - ((PASS++)) - else - echo -en "\e[41m[FAIL]\e[0m ${user}'s Bash History should be cleared to prevent sensitive information from leaking\n" - ((FAIL++)) - STATUS=2 - - fi - echo -en "\n\n" - fi - fi - done -} -function checkFirewall { - - if [[ $OS == "Ubuntu" ]]; then - fw="ufw" - ufwa=$(ufw status | head -1 | sed -e "s/^Status:\ //") - if [[ $ufwa == "active" ]]; then - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - elif [[ $OS == "CentOS Linux" ]]; then - if [ -f /usr/lib/systemd/system/csf.service ]; then - fw="csf" - if [[ $(systemctl status $fw >/dev/null 2>&1) ]]; then - - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - elif cmdExists "firewall-cmd"; then - if [[ $(systemctl is-active firewalld >/dev/null 2>&1 && echo 1 || echo 0) ]]; then - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - else - fw="firewalld" - if [[ $(systemctl is-active firewalld >/dev/null 2>&1 && echo 1 || echo 0) ]]; then - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - fi - elif [[ $OS =~ Debian.* ]]; then - # user could be using a number of different services for managing their firewall - # we will check some of the most common - if cmdExists 'ufw'; then - fw="ufw" - ufwa=$(ufw status | head -1 | sed -e "s/^Status:\ //") - if [[ $ufwa == "active" ]]; then - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - elif cmdExists "firewall-cmd"; then - fw="firewalld" - if [[ $(systemctl is-active --quiet $fw) ]]; then - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - else - # user could be using vanilla iptables, check if kernel module is loaded - fw="iptables" - if [[ $(lsmod | grep -q '^ip_tables' 2>/dev/null) ]]; then - FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" - ((PASS++)) - else - FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" - ((WARN++)) - fi - fi - fi - -} -function checkUpdates { - if [[ $OS == "Ubuntu" ]] || [[ $OS =~ Debian.* ]]; then - # Ensure /tmp exists and has the proper permissions before - # checking for security updates - # https://github.com/digitalocean/marketplace-partners/issues/94 - if [[ ! -d /tmp ]]; then - mkdir /tmp - fi - chmod 1777 /tmp - - echo -en "\nUpdating apt package database to check for security updates, this may take a minute...\n\n" - apt-get -y update >/dev/null - - uc=$(apt-get --just-print upgrade | grep -i "security" | wc -l) - if [[ $uc -gt 0 ]]; then - update_count=$((uc / 2)) - else - update_count=0 - fi - - if [[ $update_count -gt 0 ]]; then - echo -en "\e[41m[FAIL]\e[0m There are ${update_count} security updates available for this image that have not been installed.\n" - echo -en - echo -en "Here is a list of the security updates that are not installed:\n" - sleep 2 - apt-get --just-print upgrade | grep -i security | awk '{print $2}' | awk '!seen[$0]++' - echo -en - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n\n" - fi - elif [[ $OS == "CentOS Linux" ]]; then - echo -en "\nChecking for available security updates, this may take a minute...\n\n" - - update_count=$(yum check-update --security --quiet | wc -l) - if [[ $update_count -gt 0 ]]; then - echo -en "\e[41m[FAIL]\e[0m There are ${update_count} security updates available for this image that have not been installed.\n" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n" - ((PASS++)) - fi - else - echo "Error encountered" - exit 1 - fi - - return 1 -} -function checkCloudInit { - - if hash cloud-init 2>/dev/null; then - CI="\e[32m[PASS]\e[0m Cloud-init is installed.\n" - ((PASS++)) - else - CI="\e[41m[FAIL]\e[0m No valid verison of cloud-init was found.\n" - ((FAIL++)) - STATUS=2 - fi - return 1 -} -function checkMongoDB { - # Check if MongoDB is installed - # If it is, verify the version is allowed (non-SSPL) - - if [[ $OS == "Ubuntu" ]] || [[ $OS =~ Debian.* ]]; then - - if [[ -f "/usr/bin/mongod" ]]; then - version=$(/usr/bin/mongod --version --quiet | grep "db version" | sed -e "s/^db\ version\ v//") - - if version_gt $version 4.0.0; then - if version_gt $version 4.0.3; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present, ${version}" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed, ${version} is not under the SSPL" - ((PASS++)) - fi - else - if version_gt $version 3.6.8; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present, ${version}" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed, ${version} is not under the SSPL" - ((PASS++)) - fi - fi - - else - echo -en "\e[32m[PASS]\e[0m MongoDB is not installed" - ((PASS++)) - fi - - elif [[ $OS == "CentOS Linux" ]]; then - - if [[ -f "/usr/bin/mongod" ]]; then - version=$(/usr/bin/mongod --version --quiet | grep "db version" | sed -e "s/^db\ version\ v//") - - if version_gt $version 4.0.0; then - if version_gt $version 4.0.3; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed is not under the SSPL" - ((PASS++)) - fi - else - if version_gt $version 3.6.8; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed is not under the SSPL" - ((PASS++)) - fi - fi - - else - echo -en "\e[32m[PASS]\e[0m MongoDB is not installed" - ((PASS++)) - fi - - else - echo "ERROR: Unable to identify distribution" - ((FAIL++)) - STATUS 2 - return 1 - fi - -} - -function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } - -clear -echo "DigitalOcean Marketplace Image Validation Tool ${VERSION}" -echo "Executed on: ${RUNDATE}" -echo "Checking local system for Marketplace compatibility..." - -getDistro - -echo -en "\n\e[1mDistribution:\e[0m ${OS}\n" -echo -en "\e[1mVersion:\e[0m ${VER}\n\n" - -ost=0 -osv=0 - -if [[ $OS == "Ubuntu" ]]; then - ost=1 - if [[ $VER == "24.04" ]]; then - osv=1 - elif [[ $VER == "18.04" ]]; then - osv=1 - elif [[ $VER == "16.04" ]]; then - osv=1 - else - osv=0 - fi - -elif [[ $OS =~ Debian.* ]]; then - ost=1 - case "$VER" in - 9) - osv=1 - ;; - 10) - osv=1 - ;; - *) - osv=2 - ;; - esac - -elif [[ $OS == "CentOS Linux" ]]; then - ost=1 - if [[ $VER == "8" ]]; then - osv=1 - elif [[ $VER == "7" ]]; then - osv=1 - elif [[ $VER == "6" ]]; then - osv=1 - else - osv=2 - fi -else - ost=0 -fi - -if [[ $ost == 1 ]]; then - echo -en "\e[32m[PASS]\e[0m Supported Operating System Detected: ${OS}\n" - ((PASS++)) -else - echo -en "\e[41m[FAIL]\e[0m ${OS} is not a supported Operating System\n" - ((FAIL++)) - STATUS=2 -fi - -if [[ $osv == 1 ]]; then - echo -en "\e[32m[PASS]\e[0m Supported Release Detected: ${VER}\n" - ((PASS++)) -elif [[ $ost == 1 ]]; then - echo -en "\e[41m[FAIL]\e[0m ${OS} ${VER} is not a supported Operating System Version\n" - ((FAIL++)) - STATUS=2 -else - echo "Exiting..." - exit 1 -fi - -checkCloudInit - -echo -en "${CI}" - -checkFirewall - -echo -en "${FW_VER}" - -checkUpdates - -loadPasswords - -checkLogs - -echo -en "\n\nChecking all user-created accounts...\n" -checkUsers - -echo -en "\n\nChecking the root account...\n" -checkRoot - -checkAgent - -checkMongoDB - -# Summary -echo -en "\n\n---------------------------------------------------------------------------------------------------\n" - -if [[ $STATUS == 0 ]]; then - echo -en "Scan Complete.\n\e[32mAll Tests Passed!\e[0m\n" -elif [[ $STATUS == 1 ]]; then - echo -en "Scan Complete. \n\e[93mSome non-critical tests failed. Please review these items.\e[0m\e[0m\n" -else - echo -en "Scan Complete. \n\e[41mOne or more tests failed. Please review these items and re-test.\e[0m\n" -fi -echo "---------------------------------------------------------------------------------------------------" -echo -en "\e[1m${PASS} Tests PASSED\e[0m\n" -echo -en "\e[1m${WARN} WARNINGS\e[0m\n" -echo -en "\e[1m${FAIL} Tests FAILED\e[0m\n" -echo -en "---------------------------------------------------------------------------------------------------\n" - -if [[ $STATUS == 0 ]]; then - echo -en "We did not detect any issues with this image. Please be sure to manually ensure that all software installed on the base system is functional, secure and properly configured (or facilities for configuration on first-boot have been created).\n\n" - exit 0 -elif [[ $STATUS == 1 ]]; then - echo -en "Please review all [WARN] items above and ensure they are intended or resolved. If you do not have a specific requirement, we recommend resolving these items before image submission\n\n" - exit 0 -else - echo -en "Some critical tests failed. These items must be resolved and this scan re-run before you submit your image to the DigitalOcean Marketplace.\n\n" - exit 1 -fi diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh deleted file mode 100644 index 8a69f7156a..0000000000 --- a/scripts/nix-provision.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# shellcheck shell=bash - -set -o errexit -set -o pipefail -set -o xtrace - -exec 1>&2 - -function install_packages { - # Setup Ansible on host VM - sudo apt-get update && sudo apt-get install -y software-properties-common - - # Install EC2-specific packages that were deferred from stage 1 - # These packages have post-install scripts that need EC2 metadata service access - # which only works on a real running EC2 instance (not in chroot) - sudo apt-get install -y ec2-hibinit-agent ec2-instance-connect hibagent - - # Manually add GPG key with explicit keyserver - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 93C4A3FD7BB9C367 - - # Add repository and install - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # sudo add-apt-repository --yes ppa:ansible/ansible - # sudo apt-get update - sudo apt-get install -y ansible - - ansible-galaxy collection install community.general -} - -function install_nix() { - sudo su -c "sh <(curl -L https://releases.nixos.org/nix/nix-2.34.6/install) --yes --daemon --nix-extra-conf-file /dev/stdin <