From 722b1d19d39e4ab822941b4d074f32e8d20cd557 Mon Sep 17 00:00:00 2001 From: solidDoWant Date: Sun, 26 Apr 2026 19:11:22 +0000 Subject: [PATCH] feat: add `wal2json` container image This plugin supports extracting WAL changes and converting them to JSON format. Closes #77 Assisted-by: Claude Opus 4.7 Signed-off-by: solidDoWant --- wal2json/Dockerfile | 26 ++++++ wal2json/README.md | 116 +++++++++++++++++++++++++++ wal2json/metadata.hcl | 33 ++++++++ wal2json/test/chainsaw-test.yaml | 36 +++++++++ wal2json/test/check-slot-assert.yaml | 6 ++ wal2json/test/check-slot.yaml | 28 +++++++ wal2json/test/cluster-assert.yaml | 8 ++ wal2json/test/cluster.yaml | 16 ++++ 8 files changed, 269 insertions(+) create mode 100644 wal2json/Dockerfile create mode 100644 wal2json/README.md create mode 100644 wal2json/metadata.hcl create mode 100644 wal2json/test/chainsaw-test.yaml create mode 100644 wal2json/test/check-slot-assert.yaml create mode 100644 wal2json/test/check-slot.yaml create mode 100644 wal2json/test/cluster-assert.yaml create mode 100644 wal2json/test/cluster.yaml diff --git a/wal2json/Dockerfile b/wal2json/Dockerfile new file mode 100644 index 00000000..bc24cbb1 --- /dev/null +++ b/wal2json/Dockerfile @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright © contributors to CloudNativePG, established as CloudNativePG a Series of LF Projects, LLC. +# SPDX-License-Identifier: Apache-2.0 + +ARG BASE=ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie +FROM $BASE AS builder + +ARG PG_MAJOR +ARG EXT_VERSION + +USER 0 + +# Install extension via `apt-get` +RUN apt-get update && apt-get install -y --no-install-recommends \ + "postgresql-${PG_MAJOR}-wal2json=${EXT_VERSION}" + +FROM scratch +ARG PG_MAJOR + +# Licenses +COPY --from=builder /usr/share/doc/postgresql-${PG_MAJOR}-wal2json/copyright /licenses/postgresql-${PG_MAJOR}-wal2json/ + +# Libraries +COPY --from=builder /usr/lib/postgresql/${PG_MAJOR}/lib/wal2json* /lib/ +COPY --from=builder /usr/lib/postgresql/${PG_MAJOR}/lib/bitcode/ /lib/bitcode/ + +USER 65532:65532 diff --git a/wal2json/README.md b/wal2json/README.md new file mode 100644 index 00000000..5f3fa636 --- /dev/null +++ b/wal2json/README.md @@ -0,0 +1,116 @@ +# wal2json + + +[wal2json](https://github.com/eulerto/wal2json) is a PostgreSQL **logical +decoding output plugin**. It reads the write-ahead log (WAL) of a database via +a logical replication slot and emits the changes as JSON, which makes it a +common building block for change data capture (CDC) pipelines (e.g. Debezium, +custom replication tooling, or audit streams). + +Because `wal2json` is a logical decoding output plugin rather than a regular +extension, it does **not** expose a `CREATE EXTENSION` object. The plugin is +loaded on demand by PostgreSQL when a replication slot is created with the +`wal2json` output plugin. + +This image provides a convenient way to deploy and manage `wal2json` with +[CloudNativePG](https://cloudnative-pg.io/). + +## Usage + +### 1. Add the wal2json extension image to your Cluster + +Define the `wal2json` extension under the `postgresql.extensions` section of +your `Cluster` resource. Logical decoding requires `wal_level` to be set to +`logical`: + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cluster-wal2json +spec: + imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie + instances: 1 + + storage: + size: 1Gi + + postgresql: + parameters: + wal_level: logical + extensions: + - name: wal2json + image: + # renovate: suite=trixie-pgdg depName=postgresql-18-wal2json + reference: ghcr.io/cloudnative-pg/wal2json:2.6-18-trixie +``` + +> [!NOTE] +> Unlike most extensions, `wal2json` is a logical decoding output plugin and +> does not require a `Database` resource with `CREATE EXTENSION`. The plugin +> is referenced directly when a logical replication slot is created. + +### 2. Create a logical replication slot using `wal2json` + +Once the cluster is ready, connect to the database with `psql` and create a +logical replication slot that uses the `wal2json` output plugin: + +```sql +SELECT * FROM pg_create_logical_replication_slot('wal2json_slot', 'wal2json'); +``` + +Some applications (for example, Teleport), may do this automatically. + +### 3. Verify installation + +Consume changes from the slot to confirm the plugin is loaded correctly. After +producing some activity (e.g. `CREATE TABLE t (id int); INSERT INTO t VALUES +(1);`), peek at the slot: + +```sql +SELECT data +FROM pg_logical_slot_peek_changes('wal2json_slot', NULL, NULL); +``` + +You should see the changes emitted as JSON documents. + +When you are done, drop the slot to release resources: + +```sql +SELECT pg_drop_replication_slot('wal2json_slot'); +``` + +## Contributors + +This extension is maintained by: + +- FirstName LastName (@GitHub_Handle) + +The maintainers are responsible for: + +- Monitoring upstream releases and security vulnerabilities. +- Ensuring compatibility with supported PostgreSQL versions. +- Reviewing and merging contributions specific to this extension's container + image and lifecycle. + +--- + +## Licenses and Copyright + +`wal2json`: + +- **Copyright:** (c) 2013-2018 Euler Taveira de Oliveira +- **License:** BSD 3-Clause License + +All relevant license and copyright information for the `wal2json` extension +and its dependencies are bundled within the image at: + +```text +/licenses/ +``` + +By using this image, you agree to comply with the terms of the licenses +contained therein. diff --git a/wal2json/metadata.hcl b/wal2json/metadata.hcl new file mode 100644 index 00000000..3675cd1b --- /dev/null +++ b/wal2json/metadata.hcl @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright © contributors to CloudNativePG, established as CloudNativePG a Series of LF Projects, LLC. +# SPDX-License-Identifier: Apache-2.0 +metadata = { + name = "wal2json" + sql_name = "wal2json" + image_name = "wal2json" + licenses = ["BSD-3-Clause"] + shared_preload_libraries = [] + postgresql_parameters = {} + extension_control_path = [] + dynamic_library_path = [] + ld_library_path = [] + bin_path = [] + env = {} + auto_update_os_libs = false + required_extensions = [] + create_extension = false + + versions = { + bookworm = { + "18" = { + // renovate: suite=bookworm-pgdg depName=postgresql-18-wal2json + package = "2.6-3.pgdg12+1" + } + } + trixie = { + "18" = { + // renovate: suite=trixie-pgdg depName=postgresql-18-wal2json + package = "2.6-3.pgdg13+1" + } + } + } +} diff --git a/wal2json/test/chainsaw-test.yaml b/wal2json/test/chainsaw-test.yaml new file mode 100644 index 00000000..69c8d092 --- /dev/null +++ b/wal2json/test/chainsaw-test.yaml @@ -0,0 +1,36 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: verify-wal2json-output-plugin +spec: + timeouts: + apply: 5s + assert: 3m + delete: 30s + description: >- + Verify that the wal2json shared library is loadable by PostgreSQL by + creating and dropping a logical replication slot using the wal2json output + plugin. Slot creation forces PostgreSQL to dlopen wal2json.so, so this is + the actual end-to-end install check for an output plugin (which has no + CREATE EXTENSION counterpart). + steps: + - name: Provision PostgreSQL Cluster + description: >- + Deploy a Cluster resource using the built wal2json image with superuser + access enabled so the slot-creation Job can call + pg_create_logical_replication_slot. + try: + - apply: + file: cluster.yaml + - assert: + file: cluster-assert.yaml + + - name: Create logical replication slot using wal2json + description: >- + Run a Job that creates and drops a logical replication slot with the + 'wal2json' output plugin, exercising dlopen of wal2json.so. + try: + - apply: + file: check-slot.yaml + - assert: + file: check-slot-assert.yaml diff --git a/wal2json/test/check-slot-assert.yaml b/wal2json/test/check-slot-assert.yaml new file mode 100644 index 00000000..1230e21c --- /dev/null +++ b/wal2json/test/check-slot-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: wal2json-slot-test +status: + succeeded: 1 diff --git a/wal2json/test/check-slot.yaml b/wal2json/test/check-slot.yaml new file mode 100644 index 00000000..794bf808 --- /dev/null +++ b/wal2json/test/check-slot.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: wal2json-slot-test +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: slot-test + env: + - name: SU_URI + valueFrom: + secretKeyRef: + name: (join('-', [$values.name, 'superuser'])) + key: uri + image: alpine/psql:latest + command: ['sh', '-c'] + args: + - | + set -e + SU_URI=$(echo $SU_URI | sed "s|/\*|/|") + SLOT="wal2json_chainsaw_test" + psql "$SU_URI" -v ON_ERROR_STOP=1 -tAc \ + "SELECT slot_name FROM pg_create_logical_replication_slot('${SLOT}', 'wal2json')" + psql "$SU_URI" -v ON_ERROR_STOP=1 -tAc \ + "SELECT pg_drop_replication_slot('${SLOT}')" + echo "wal2json output plugin loaded successfully" diff --git a/wal2json/test/cluster-assert.yaml b/wal2json/test/cluster-assert.yaml new file mode 100644 index 00000000..aa221230 --- /dev/null +++ b/wal2json/test/cluster-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ($values.name) +status: + readyInstances: 1 + phase: Cluster in healthy state + image: ($values.pg_image) diff --git a/wal2json/test/cluster.yaml b/wal2json/test/cluster.yaml new file mode 100644 index 00000000..434953f4 --- /dev/null +++ b/wal2json/test/cluster.yaml @@ -0,0 +1,16 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ($values.name) +spec: + imageName: ($values.pg_image) + instances: 1 + enableSuperuserAccess: true + + storage: + size: 1Gi + + postgresql: + parameters: ($values.postgresql_parameters) + shared_preload_libraries: ($values.shared_preload_libraries) + extensions: ($values.extensions)