Add a workflow to test package installation for Citus packages

.github/workflows/test-package-installation.yml

@@ -0,0 +1,93 @@
+name: Test Latest Citus Package Installation on Supported PostgreSQL Versions
+
+on:
+  workflow_dispatch:
+
+env:
+  # Branch whose build-package.yml, postgres-matrix.yml and pkgvars files are the source of truth
+  SOURCE_BRANCH: "all-citus"
+
+jobs:
+  discover:
+    name: Discover platforms & versions
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.build_matrix.outputs.matrix }}
+      citus_version: ${{ steps.resolve_versions.outputs.citus_version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Fetch source branch
+        run: git fetch origin "${SOURCE_BRANCH}" --depth=1
+
+      - name: Extract platform list
+        id: extract_platforms
+        run: |
+          git show "origin/${SOURCE_BRANCH}:.github/workflows/build-package.yml" \
+            > /tmp/build-package.yml
+
+          platforms=$(
+            yq -o=json '.jobs.build_package.strategy.matrix.platform' /tmp/build-package.yml \
+            | jq -c '.'
+          )
+
+          echo "Discovered platforms: ${platforms}"
+          echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve Citus & PG versions
+        id: resolve_versions
+        run: |
+          git show "origin/${SOURCE_BRANCH}:pkgvars" > /tmp/pkgvars
+          git show "origin/${SOURCE_BRANCH}:postgres-matrix.yml" > /tmp/postgres-matrix.yml
+
+          # Extract version from pkglatest (e.g. "14.0.0.citus-1" -> "14.0.0")
+          citus_version=$(grep '^pkglatest=' /tmp/pkgvars | sed 's/^pkglatest=//;s/\.citus-.*//')
+
+          # postgres-matrix.yml keys use major.minor (e.g. "14.0"), strip patch
+          citus_major_minor=$(echo "${citus_version}" | cut -d. -f1,2)
+
+          pg_versions=$(
+            yq -o=json \
+              ".version_matrix[-1].\"${citus_major_minor}\".postgres_versions" \
+              /tmp/postgres-matrix.yml \
+            | jq -c '[.[] | tostring]'
+          )
+
+          echo "Latest Citus version: ${citus_version}"
+          echo "Supported PG versions: ${pg_versions}"
+          echo "citus_version=${citus_version}" >> "$GITHUB_OUTPUT"
+          echo "pg_versions=${pg_versions}" >> "$GITHUB_OUTPUT"
+
+      - name: Build matrix
+        id: build_matrix
+        run: |
+          matrix=$(jq -cn \
+            --argjson platforms '${{ steps.extract_platforms.outputs.platforms }}' \
+            --argjson pg_versions '${{ steps.resolve_versions.outputs.pg_versions }}' \
+            '{"platform": $platforms, "pg_version": $pg_versions}')
+          echo "Matrix: ${matrix}"
+          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
+
+  test_package:
+    name: "PG${{ matrix.pg_version }}/Citus${{ needs.discover.outputs.citus_version }} - ${{ matrix.platform }}"
+    needs: discover
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.discover.outputs.matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ">=3.10"
+
+      - name: Run package installation test
+        run: |
+          python3 test_package_installation/run_test.py \
+            "${{ matrix.platform }}" \
+            "${{ matrix.pg_version }}" \
+            "${{ needs.discover.outputs.citus_version }}"

.gitignore

@@ -7,3 +7,5 @@
 
 # package output directory
 /packages
+
+.vscode

test_package_installation/README.md

@@ -0,0 +1,35 @@
+# Package Installation Tests
+
+Verifies that Citus packages can be installed from the package repository on all supported platforms.
+
+Each test:
+* spins up a Docker container for the target OS
+* installs the PostgreSQL and Citus packages
+* sets up a cluster and runs very basic tests
+* runs `make check-multi` regression tests from the Citus repository against the installed Citus version
+
+## Running locally
+
+Requires **Python 3.10+** and **Docker**. No additional Python packages needed.
+
+```bash
+python3 test_package_installation/run_test.py <platform> <pg_major_version> <citus_full_version>
+```
+
+Examples:
+
+```bash
+python3 test_package_installation/run_test.py ubuntu/noble 18 14.0.0
+python3 test_package_installation/run_test.py el/9 17 14.0.0
+python3 test_package_installation/run_test.py debian/bookworm 16 14.0.0
+```
+
+## Running via GitHub Actions
+
+You can manually trigger the workflow using [this GitHub Actions tab](https://github.com/citusdata/packaging/actions/workflows/test-package-installation.yml).
+
+It automatically discovers the supported platforms, latest Citus version that we published
+packages for, and compatible PostgreSQL versions from the `all-citus` branch, using
+[pkgvars](https://github.com/citusdata/packaging/blob/all-citus/pkgvars),
+[.github/workflows/build-package.yml](https://github.com/citusdata/packaging/blob/all-citus/.github/workflows/build-package.yml) and
+[postgres-matrix.yml](https://github.com/citusdata/packaging/blob/all-citus/postgres-matrix.yml) files.

test_package_installation/common/run_multi_schedule.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Runs "make check-multi" (multi_schedule).
+#
+# Expects the Citus source to be cloned at /citus and pg_regress
+# to be available in the PostgreSQL binary path.
+#
+# Usage: run_multi_schedule.sh <pg_binary_path>
+# Example: run_multi_schedule.sh /usr/lib/postgresql/18/bin
+
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "Error: some arguments are missing." >&2
+    echo "Usage: $0 <pg_binary_path>" >&2
+    exit 1
+fi
+
+PG_BIN="$1"
+
+## suppress some of the regression test failures
+
+# On rpm based distros, Postgres package might be linked against a newer ICU
+# than the one used to generate expected test output. To avoid test failures
+# due to this, patch the expected output to match the newer ICU version.
+# Without this, today pg18.sql fails.
+echo 's/und-u-kc-true-ks-level1/und-u-kc-ks-level1/g' >> /citus/src/test/regress/bin/normalize.sed
+
+## run "make check-multi"
+
+export PATH="${PG_BIN}:${PATH}"
+
+cd /citus
+
+# we don't need lz4 and zstd as none of the tests in multi_schedule depend on them
+./configure PG_CONFIG="${PG_BIN}/pg_config" --without-libcurl --without-lz4 --without-zstd
+
+chown -R postgres:postgres /citus
+
+echo "Running make check-multi ..."
+
+if ! su - postgres -c "make check-multi -C /citus/src/test/regress"; then
+    echo "make check-multi failed. Printing regression.diffs ..."
+    cat /citus/src/test/regress/regression.diffs
+    exit 1
+fi
+
+echo "make check-multi completed successfully."

test_package_installation/common/test_basics.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Sets up a multi-node Citus cluster, creates a distributed table,
+# rebalances shards, and verifies even distribution.
+#
+# Note: need to run this as root
+#
+# Usage: test_basics.sh <expected_pg_major_version> <expected_citus_full_version> <pg_binary_path>
+# Example: test_basics.sh 18 14.0.0 /usr/lib/postgresql/17/bin
+
+set -euo pipefail
+
+if [ "$#" -ne 3 ]; then
+    echo "Error: some arguments are missing." >&2
+    echo "Usage: $0 <expected_pg_major_version> <expected_citus_full_version> <pg_binary_path>" >&2
+    exit 1
+fi
+
+EXPECTED_PG_MAJOR_VERSION="$1"
+EXPECTED_CITUS_FULL_VERSION="$2"
+PG_BIN="$3"
+
+## constants and helpers
+
+NODE_COUNT=3
+SHARDS_PER_NODE=5
+FIRST_NODE_PORT=9700
+
+run_psql_cmd() {
+    if [ "$#" -ne 2 ]; then
+        echo "Error: some arguments are missing." >&2
+        echo "Usage: run_psql_cmd <port_number> <sql_command>" >&2
+        exit 1
+    fi
+
+    PORT_NUMBER="$1"
+    SQL_COMMAND="$2"
+
+    su - postgres -c "${PG_BIN}/psql -X -qAt -p ${PORT_NUMBER} -c \"${SQL_COMMAND}\""
+}
+
+## initialize all nodes and create citus extension on them
+
+echo "Initializing nodes and creating citus extension on them ..."
+
+mkdir -p /test_cluster
+chown postgres:postgres /test_cluster
+chmod 755 /test_cluster
+
+for i in $(seq 0 $((NODE_COUNT - 1))); do
+    node_name="node${i}"
+    echo "Creating node: $node_name"
+
+    su - postgres -c "mkdir -p /test_cluster/${node_name}"
+
+    port_number=$((FIRST_NODE_PORT + i))
+
+    su - postgres -c "${PG_BIN}/initdb -D /test_cluster/${node_name}"
+    su - postgres -c "echo \"shared_preload_libraries = 'citus'\" >> /test_cluster/${node_name}/postgresql.conf"
+    su - postgres -c "${PG_BIN}/pg_ctl -D /test_cluster/${node_name} -o \"-p $port_number\" -l /test_cluster/${node_name}/logfile start"
+    run_psql_cmd "$port_number" "CREATE EXTENSION citus;"
+done
+
+## verify PG version
+
+pg_version=$(run_psql_cmd "$FIRST_NODE_PORT" "SELECT version();")
+if [[ "$pg_version" != "PostgreSQL ${EXPECTED_PG_MAJOR_VERSION}."* ]]; then
+    echo "Error: PostgreSQL version verification failed. Expected version to start with 'PostgreSQL ${EXPECTED_PG_MAJOR_VERSION}.', got: $pg_version" >&2
+    exit 1
+else
+    echo "Verified PostgreSQL version: $pg_version"
+fi
+
+## verify Citus extension version
+
+citus_version=$(run_psql_cmd "$FIRST_NODE_PORT" "SHOW citus.version;")
+if [[ "$citus_version" != "${EXPECTED_CITUS_FULL_VERSION}"* ]]; then
+    echo "Error: Citus version verification failed. Expected version to start with '${EXPECTED_CITUS_FULL_VERSION}', got: $citus_version" >&2
+    exit 1
+else
+    echo "Verified Citus version: $citus_version"
+fi
+
+## create a distributed table
+
+echo "Creating a distributed table ..."
+
+shard_count=$((NODE_COUNT * SHARDS_PER_NODE))
+run_psql_cmd "$FIRST_NODE_PORT" "CREATE TABLE test_table (id INT); SELECT create_distributed_table('test_table', 'id', shard_count => $shard_count);"
+
+## add worker nodes
+
+echo "Adding worker nodes ..."
+
+for i in $(seq 1 $((NODE_COUNT - 1))); do
+    port_number=$((FIRST_NODE_PORT + i))
+    run_psql_cmd "$FIRST_NODE_PORT" "SELECT citus_add_node('localhost', $port_number);"
+done
+
+## run rebalancer
+
+echo "Running shard rebalancer ..."
+
+run_psql_cmd "$FIRST_NODE_PORT" "SELECT rebalance_table_shards(shard_transfer_mode =>'block_writes');"
+
+## verify that shards are distributed evenly across nodes
+
+echo "Verifying that shards are distributed evenly across nodes ..."
+
+result=$(
+    run_psql_cmd "$FIRST_NODE_PORT" "
+        SELECT bool_and(cnt = $SHARDS_PER_NODE)
+        FROM (
+            SELECT nodeport,
+                   COUNT(shardid) AS cnt
+            FROM pg_dist_node
+            LEFT JOIN pg_dist_shard_placement USING (nodeport)
+            GROUP BY nodeport
+        ) t;
+    "
+)
+
+if [ "$result" = "t" ]; then
+    echo "Verified cluster creation, distributed table creation and shard rebalancing. Shards are distributed evenly across nodes."
+else
+    echo "Error: Failed. Shards are not distributed evenly across nodes, see below for details:" >&2
+    run_psql_cmd "$FIRST_NODE_PORT" "
+        SELECT nodeport,
+               COUNT(shardid) AS cnt
+        FROM pg_dist_node
+        LEFT JOIN pg_dist_shard_placement USING (nodeport)
+        GROUP BY nodeport;
+    "
+    exit 1
+fi

test_package_installation/deb/install_package.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Installs PostgreSQL and Citus packages on a Debian/Ubuntu system.
+#
+# Note: need to run this as root
+#
+# Usage: install_package.sh <pg_major_version> <citus_full_version>
+# Example: install_package.sh 18 14.0.0
+
+set -euo pipefail
+
+if [ "$#" -ne 2 ]; then
+    echo "Error: some arguments are missing." >&2
+    echo "Usage: $0 <pg_major_version> <citus_full_version>" >&2
+    exit 1
+fi
+
+PG_MAJOR_VERSION="$1"
+CITUS_FULL_VERSION="$2"
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get -y update
+
+## install postgres
+
+echo "Installing PostgreSQL ..."
+
+apt-get -y install curl ca-certificates
+
+install -d /usr/share/postgresql-common/pgdg
+curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
+. /etc/os-release
+sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main' > /etc/apt/sources.list.d/pgdg.list"
+
+apt -y update
+apt-get -y install postgresql-${PG_MAJOR_VERSION}
+
+## install citus
+
+echo "Installing Citus ..."
+
+curl --fail https://install.citusdata.com/community/deb.sh > add-citus-repo.sh
+bash add-citus-repo.sh
+
+citus_major_minor_version="${CITUS_FULL_VERSION%.*}"
+apt-get -y install postgresql-${PG_MAJOR_VERSION}-citus-${citus_major_minor_version}=${CITUS_FULL_VERSION}.citus-1
+
+echo "PostgreSQL and Citus packages installed successfully."