From 9092fb568f2a1977cd18b4a578ebda983eb3ae7a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 27 Apr 2026 10:55:24 +0200 Subject: [PATCH 1/4] feat: Add support for the PostgreSQL connector --- CHANGELOG.md | 6 ++ Cargo.nix | 18 +++--- crate-hashes.json | 18 +++--- .../usage-guide/catalogs/black-hole.adoc | 1 + .../usage-guide/catalogs/delta-lake.adoc | 1 + .../pages/usage-guide/catalogs/generic.adoc | 1 + .../usage-guide/catalogs/google-sheets.adoc | 1 + .../pages/usage-guide/catalogs/hive.adoc | 7 +++ .../pages/usage-guide/catalogs/index.adoc | 1 + .../usage-guide/catalogs/postgresql.adoc | 32 +++++++++++ .../pages/usage-guide/catalogs/tpcds.adoc | 1 + .../pages/usage-guide/catalogs/tpch.adoc | 1 + docs/modules/trino/partials/nav.adoc | 1 + extra/crds.yaml | 37 ++++++++++++ rust/operator-binary/src/catalog/config.rs | 56 +++++-------------- rust/operator-binary/src/catalog/mod.rs | 6 ++ .../operator-binary/src/catalog/postgresql.rs | 47 ++++++++++++++++ rust/operator-binary/src/crd/affinity.rs | 2 + rust/operator-binary/src/crd/catalog/mod.rs | 5 ++ .../src/crd/catalog/postgresql.rs | 13 +++++ tests/templates/kuttl/commons/check-s3.py | 23 ++++---- .../kuttl/delta/08-install-hive.yaml.j2 | 9 +-- .../opa-authorization/03-install-hive.yaml.j2 | 9 +-- .../kuttl/smoke/08-install-hive.yaml.j2 | 9 +-- .../kuttl/smoke/10-install-trino.yaml.j2 | 19 ++++++- .../kuttl/smoke_aws/08-install-hive.yaml.j2 | 9 +-- 26 files changed, 243 insertions(+), 90 deletions(-) create mode 100644 docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc create mode 100644 rust/operator-binary/src/catalog/postgresql.rs create mode 100644 rust/operator-binary/src/crd/catalog/postgresql.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c581a547..52375902a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Added support for the [PostgreSQL connector](https://trino.io/docs/current/connector/postgresql.html) using the new generic database connection mechanism. + Previously, users had to use the `generic` connector ([#XXX]). + ### Changed - BREAKING: `configOverrides` now only accepts the supported config file names @@ -26,6 +31,7 @@ All notable changes to this project will be documented in this file. [#869]: https://github.com/stackabletech/trino-operator/pull/869 [#876]: https://github.com/stackabletech/trino-operator/pull/876 [#878]: https://github.com/stackabletech/trino-operator/pull/878 +[#XXX]: https://github.com/stackabletech/trino-operator/pull/XXX ## [26.3.0] - 2026-03-16 diff --git a/Cargo.nix b/Cargo.nix index 856fbaaa7..951276451 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4863,7 +4863,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "k8s_version"; authors = [ @@ -9492,7 +9492,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_certs"; authors = [ @@ -9595,7 +9595,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_operator"; authors = [ @@ -9775,7 +9775,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9810,7 +9810,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_shared"; authors = [ @@ -9891,7 +9891,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_telemetry"; authors = [ @@ -10102,7 +10102,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_versioned"; authors = [ @@ -10146,7 +10146,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10214,7 +10214,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 33e75762e..1f3488f2f 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#k8s-version@0.1.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-certs@0.4.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator-derive@0.3.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator@0.110.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-shared@0.1.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-telemetry@0.6.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned-macros@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-webhook@0.9.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#k8s-version@0.1.3": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-certs@0.4.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator-derive@0.3.1": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator@0.110.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-shared@0.1.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-telemetry@0.6.3": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned-macros@0.9.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned@0.9.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-webhook@0.9.1": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc b/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc index 55688f7c2..8b48ec056 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc @@ -4,6 +4,7 @@ Primarily the https://trino.io/docs/current/connector/blackhole.html[Black Hole It works like the `/dev/null` device on Unix-like operating systems for data writing and like `/dev/null` or `/dev/zero` for data reading. == Example Black Hole catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc b/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc index 91c7d9a64..ba1661148 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc @@ -30,5 +30,6 @@ spec: ---- == Connect to S3 store or HDFS + The Delta Lake connector connects to S3 or HDFS in the same way the xref:usage-guide/catalogs/hive.adoc[] connector does. Refer to that documentation for access configuration. diff --git a/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc b/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc index 420acab8b..427d0b3a4 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc @@ -8,6 +8,7 @@ In case the Stackable operator for Trino does not support a specific connector y This is how you can e.g. use the {trino-psql-connector}[PostgreSQL connector]: == Example generic catalog properties + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc b/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc index a7a9d266f..cb77ca63f 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc @@ -6,6 +6,7 @@ It needs a service-user to access the Google APIs. Consult the {trino-google-sheets-connector}[official documentation] on how to use the Google Sheets connector. == Example Google sheets catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc b/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc index 556636dd1..db3de6ac2 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc @@ -5,6 +5,7 @@ The Apache Hive connector allows Trino to connect to a Hive metastore and query Deploy a Hive Stacklet with the xref:hive:index.adoc[]. == Example Hive catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 @@ -34,6 +35,7 @@ spec: <4> Use `configOverrides` to add arbitrary properties to the Trino catalog configuration == Connect to S3 store + The Hive connector can connect to an S3 store as follows: [source,yaml] @@ -58,6 +60,7 @@ See xref:concepts:s3.adoc[] for details about S3 connections. IMPORTANT: Make sure that the underlying Hive metastore also has access to the S3 store, because it will e.g. check if the directory exists when creating tables. == Connect to HDFS cluster + The hive connector can connect to an HDFS operated by Stackable as follows: [source,yaml] @@ -72,10 +75,12 @@ spec: IMPORTANT: Make sure that the underlying Hive metastore also has access to the HDFS, because it will e.g. check if the directory exists when creating tables. == Adding unmanaged Hive clusters + You can add connect Trino to Hive catalogs from systems that are not managed by Stackable, including Hive running on existing Hadoop clusters. Unmanaged Hive instances can be defined by creating a ConfigMap containing the configuration for the remote Hive Metastore and HDFS or S3 storage services. === Create a Hive Metastore configMap + The Hive metastore ConfigMap contains the URL for the metastore's thrift endpoint. [source,yaml] ---- @@ -88,6 +93,7 @@ data: ---- === Create a HDFS configMap + When the Hive data is stored on HDFS you will need to provide a ConfigMap containing the HDFS configuration. To do this take the `core-site.xml` and `hdfs-site.xml` from your Hadoop cluster and create a ConfigMap with the keys `core-site.xml` and `hdfs-site.xml`. @@ -119,6 +125,7 @@ data: ---- === Create the Trino Hive catalog + To use the unmanaged Hive metastore we define a TrinoCatalog object in the same way we would for a managed cluster, referencing the custom ConfigMap we created for Hive and HDFS. [source,yaml] diff --git a/docs/modules/trino/pages/usage-guide/catalogs/index.adoc b/docs/modules/trino/pages/usage-guide/catalogs/index.adoc index ae6b560ac..3c62ede9c 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/index.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/index.adoc @@ -4,6 +4,7 @@ Trino does not have a built-in catalog and instead provides connectors to extern This allows Trino to connect to, read from and join a wide variety of data sources. == Using catalogs + Catalogs are defined in their own resources and referenced from cluster objects. See the xref:concepts.adoc[] page for more details. diff --git a/docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc b/docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc new file mode 100644 index 000000000..185649170 --- /dev/null +++ b/docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc @@ -0,0 +1,32 @@ += PostgreSQL + +The https://trino.io/docs/current/connector/postgresql.html[PostgreSQL connector] provides connectivity to a PostgreSQL databases. + +== Example PostgreSQL catalog configuration + +[source,yaml] +---- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + # The name of the catalog as it will appear in Trino + name: postgresql + # TrinoCluster can use these labels to select which catalogs to include + labels: + trino: simple-trino +spec: + connector: + postgresql: + host: my-postgresql + port: 5432 # defaults to 5432 + database: mydatabase + credentialsSecretName: my-postgresql-credentials # Must provide the username and password keys +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-postgresql-credentials +stringData: + username: myusername + password: mypassword +---- diff --git a/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc b/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc index 454aebfb0..b7623011a 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc @@ -4,6 +4,7 @@ The https://trino.io/docs/current/connector/tpcds.html[TPC-DS connector] provide This is not a stored data set and is instead a virtual data set generated at query time using a deterministic algorithm. == Example TPC-DS catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc b/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc index 332abc658..963c9bfd0 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc @@ -4,6 +4,7 @@ The https://trino.io/docs/current/connector/tpch.html[TPC-H connector] provides This is not a stored data set and is instead a virtual data set generated at query time using a deterministic algorithm. == Example TPC-H catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/partials/nav.adoc b/docs/modules/trino/partials/nav.adoc index 453c0fa01..d90607af9 100644 --- a/docs/modules/trino/partials/nav.adoc +++ b/docs/modules/trino/partials/nav.adoc @@ -22,6 +22,7 @@ *** xref:trino:usage-guide/catalogs/google-sheets.adoc[] *** xref:trino:usage-guide/catalogs/hive.adoc[] *** xref:trino:usage-guide/catalogs/iceberg.adoc[] +*** xref:trino:usage-guide/catalogs/postgresql.adoc[] *** xref:trino:usage-guide/catalogs/tpcds.adoc[] *** xref:trino:usage-guide/catalogs/tpch.adoc[] ** xref:trino:usage-guide/operations/index.adoc[] diff --git a/extra/crds.yaml b/extra/crds.yaml index 4729af1dd..f762ef6bb 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -3577,6 +3577,8 @@ spec: - hive - required: - iceberg + - required: + - postgresql - required: - tpcds - required: @@ -4216,6 +4218,41 @@ spec: type: string type: object type: object + postgresql: + description: An [PostgreSQL](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/postgresql) connector. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object tpcds: description: A [TPC-DS](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/tpcds) connector. type: object diff --git a/rust/operator-binary/src/catalog/config.rs b/rust/operator-binary/src/catalog/config.rs index de6a18adb..8d4ef5530 100644 --- a/rust/operator-binary/src/catalog/config.rs +++ b/rust/operator-binary/src/catalog/config.rs @@ -114,48 +114,20 @@ impl CatalogConfig { .ok_or(FromTrinoCatalogError::InvalidCatalogSpec)?; let catalog_namespace = catalog.namespace(); - let mut catalog_config = match &catalog.spec.connector { - TrinoCatalogConnector::BlackHole(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::DeltaLake(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::GoogleSheet(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Generic(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Hive(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Iceberg(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Tpcds(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Tpch(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - }?; + let to_catalog_config: &dyn ToCatalogConfig = match &catalog.spec.connector { + TrinoCatalogConnector::BlackHole(black_hole_connector) => black_hole_connector, + TrinoCatalogConnector::DeltaLake(delta_lake_connector) => delta_lake_connector, + TrinoCatalogConnector::GoogleSheet(google_sheet_connector) => google_sheet_connector, + TrinoCatalogConnector::Generic(generic_connector) => generic_connector, + TrinoCatalogConnector::Hive(hive_connector) => hive_connector, + TrinoCatalogConnector::Iceberg(iceberg_connector) => iceberg_connector, + TrinoCatalogConnector::Postgresql(postgresql_connector) => postgresql_connector, + TrinoCatalogConnector::Tpcds(tpcds_connector) => tpcds_connector, + TrinoCatalogConnector::Tpch(tpch_connector) => tpch_connector, + }; + let mut catalog_config = to_catalog_config + .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) + .await?; catalog_config .properties diff --git a/rust/operator-binary/src/catalog/mod.rs b/rust/operator-binary/src/catalog/mod.rs index 63c670a63..9abc549a6 100644 --- a/rust/operator-binary/src/catalog/mod.rs +++ b/rust/operator-binary/src/catalog/mod.rs @@ -6,6 +6,7 @@ pub mod generic; pub mod google_sheet; pub mod hive; pub mod iceberg; +pub mod postgresql; pub mod tpcds; pub mod tpch; @@ -61,6 +62,11 @@ pub enum FromTrinoCatalogError { CreateS3CredentialsSecretOperatorVolume { source: stackable_operator::builder::pod::volume::SecretOperatorVolumeSourceBuilderError, }, + + #[snafu(display("failed to get PostgreSQL connection details"))] + GetPostgresConnectionDetails { + source: stackable_operator::database_connections::Error, + }, } #[async_trait] diff --git a/rust/operator-binary/src/catalog/postgresql.rs b/rust/operator-binary/src/catalog/postgresql.rs new file mode 100644 index 000000000..cb052a759 --- /dev/null +++ b/rust/operator-binary/src/catalog/postgresql.rs @@ -0,0 +1,47 @@ +use async_trait::async_trait; +use snafu::ResultExt; +use stackable_operator::{ + client::Client, database_connections::drivers::jdbc::JdbcDatabaseConnection, +}; + +use super::{FromTrinoCatalogError, ToCatalogConfig, config::CatalogConfig}; +use crate::{ + catalog::from_trino_catalog_error::GetPostgresConnectionDetailsSnafu, + crd::catalog::postgresql::PostgresqlConnector, +}; + +pub const CONNECTOR_NAME: &str = "postgresql"; + +#[async_trait] +impl ToCatalogConfig for PostgresqlConnector { + async fn to_catalog_config( + &self, + catalog_name: &str, + _catalog_namespace: Option, + _client: &Client, + _trino_version: u16, + ) -> Result { + let mut config = CatalogConfig::new(catalog_name.to_string(), CONNECTOR_NAME); + // SAFETY: `unique_database_name` must only contains ASCII letters and underscores. + let unique_database_name = format!("POSTGRESQL_{}", catalog_name.replace('-', "_")); + let jdbc_connection_details = self + .inner + .jdbc_connection_details(&unique_database_name) + .context(GetPostgresConnectionDetailsSnafu)?; + + config.add_property("connection-url", jdbc_connection_details.connection_url); + if let Some(username_env) = jdbc_connection_details.username_env { + config.add_property("connection-user", format!("${{ENV:{}}}", username_env.name)); + config.env_bindings.push(username_env); + }; + if let Some(password_env) = jdbc_connection_details.password_env { + config.add_property( + "connection-password", + format!("${{ENV:{}}}", password_env.name), + ); + config.env_bindings.push(password_env); + }; + + Ok(config) + } +} diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index cac09f421..bac46c9ad 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -32,6 +32,7 @@ pub fn get_affinity( TrinoCatalogConnector::BlackHole(_) | TrinoCatalogConnector::Generic(_) | TrinoCatalogConnector::GoogleSheet(_) + | TrinoCatalogConnector::Postgresql(_) | TrinoCatalogConnector::Tpcds(_) | TrinoCatalogConnector::Tpch(_) => None, }) @@ -59,6 +60,7 @@ pub fn get_affinity( TrinoCatalogConnector::BlackHole(_) | TrinoCatalogConnector::Generic(_) | TrinoCatalogConnector::GoogleSheet(_) + | TrinoCatalogConnector::Postgresql(_) | TrinoCatalogConnector::Tpcds(_) | TrinoCatalogConnector::Tpch(_) => None, }) diff --git a/rust/operator-binary/src/crd/catalog/mod.rs b/rust/operator-binary/src/crd/catalog/mod.rs index 2c81c98dc..c10dd80ce 100644 --- a/rust/operator-binary/src/crd/catalog/mod.rs +++ b/rust/operator-binary/src/crd/catalog/mod.rs @@ -5,6 +5,7 @@ pub mod generic; pub mod google_sheet; pub mod hive; pub mod iceberg; +pub mod postgresql; pub mod tpcds; pub mod tpch; @@ -25,6 +26,7 @@ use tpcds::TpcdsConnector; use tpch::TpchConnector; use self::delta_lake::DeltaLakeConnector; +use crate::crd::catalog::postgresql::PostgresqlConnector; #[versioned( version(name = "v1alpha1"), @@ -86,6 +88,9 @@ pub enum TrinoCatalogConnector { /// An [Apache Iceberg](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/iceberg) connector. Iceberg(IcebergConnector), + /// An [PostgreSQL](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/postgresql) connector. + Postgresql(PostgresqlConnector), + /// A [TPC-DS](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/tpcds) connector. Tpcds(TpcdsConnector), diff --git a/rust/operator-binary/src/crd/catalog/postgresql.rs b/rust/operator-binary/src/crd/catalog/postgresql.rs new file mode 100644 index 000000000..8d13c0b09 --- /dev/null +++ b/rust/operator-binary/src/crd/catalog/postgresql.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + database_connections::databases::postgresql::PostgresqlConnection, + schemars::{self, JsonSchema}, +}; + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PostgresqlConnector { + // Docs are on the struct fields + #[serde(flatten)] + pub inner: PostgresqlConnection, +} diff --git a/tests/templates/kuttl/commons/check-s3.py b/tests/templates/kuttl/commons/check-s3.py index 9c2474f6b..5749214bb 100755 --- a/tests/templates/kuttl/commons/check-s3.py +++ b/tests/templates/kuttl/commons/check-s3.py @@ -279,16 +279,17 @@ def run_query(connection, query): # Test could be improved by also testing update and deletes - # # Test postgres connection - # run_query(connection, "SHOW SCHEMAS IN postgresgeneric") - # run_query(connection, "CREATE SCHEMA IF NOT EXISTS postgresgeneric.tpch") - # run_query( - # connection, - # "CREATE TABLE IF NOT EXISTS postgresgeneric.tpch.nation AS SELECT * FROM tpch.tiny.nation", - # ) - # assert ( - # run_query(connection, "SELECT COUNT(*) FROM postgresgeneric.tpch.nation")[0][0] - # == 25 - # ) + # Test postgres connection(s) + for catalog in ["postgresql", "postgresqlgeneric"]: + run_query(connection, f"SHOW SCHEMAS IN {catalog}") + run_query(connection, f"CREATE SCHEMA IF NOT EXISTS {catalog}.tpch") + run_query( + connection, + f"CREATE TABLE IF NOT EXISTS {catalog}.tpch.nation AS SELECT * FROM tpch.tiny.nation", + ) + assert ( + run_query(connection, f"SELECT COUNT(*) FROM {catalog}.tpch.nation")[0][0] + == 25 + ) print("[SUCCESS] All tests in check-s3.py succeeded!") diff --git a/tests/templates/kuttl/delta/08-install-hive.yaml.j2 b/tests/templates/kuttl/delta/08-install-hive.yaml.j2 index 9eff5093b..d145f8006 100644 --- a/tests/templates/kuttl/delta/08-install-hive.yaml.j2 +++ b/tests/templates/kuttl/delta/08-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive-latest'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 b/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 index 9eff5093b..d145f8006 100644 --- a/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 +++ b/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive-latest'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 b/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 index 446ce72a6..647d6ea83 100644 --- a/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials hdfs: configMap: hdfs s3: diff --git a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 index d94cfb81a..3b085475f 100644 --- a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 @@ -64,7 +64,20 @@ spec: apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCatalog metadata: - name: postgresgeneric + name: postgresql + labels: + trino: trino +spec: + connector: + postgresql: + host: postgresql + database: hive + credentialsSecretName: my-postgresql-credentials-secret +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: postgresqlgeneric labels: trino: trino spec: @@ -78,7 +91,7 @@ spec: connection-user: valueFromSecret: name: my-postgresql-credentials-secret - key: user + key: username connection-password: valueFromSecret: name: my-postgresql-credentials-secret @@ -89,7 +102,7 @@ kind: Secret metadata: name: my-postgresql-credentials-secret stringData: - user: hive + username: hive password: hive --- # We need to create the TrinoCluster last, so that the ConfigMaps/Secrets it mounts are already diff --git a/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 b/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 index 41d811e77..a875f348a 100644 --- a/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials hdfs: configMap: hdfs s3: From 46101dacfd7e235783323f81223cf683e4aadcb5 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 27 Apr 2026 10:57:46 +0200 Subject: [PATCH 2/4] changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52375902a..fd0627c4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Added support for the [PostgreSQL connector](https://trino.io/docs/current/connector/postgresql.html) using the new generic database connection mechanism. - Previously, users had to use the `generic` connector ([#XXX]). + Previously, users had to use the `generic` connector ([#883]). ### Changed @@ -31,7 +31,7 @@ All notable changes to this project will be documented in this file. [#869]: https://github.com/stackabletech/trino-operator/pull/869 [#876]: https://github.com/stackabletech/trino-operator/pull/876 [#878]: https://github.com/stackabletech/trino-operator/pull/878 -[#XXX]: https://github.com/stackabletech/trino-operator/pull/XXX +[#883]: https://github.com/stackabletech/trino-operator/pull/883 ## [26.3.0] - 2026-03-16 From a3948364206cf29d20fb04b4c9ca94dc4fd19e26 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 27 Apr 2026 14:08:59 +0200 Subject: [PATCH 3/4] Use to_uppercase --- rust/operator-binary/src/catalog/postgresql.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/catalog/postgresql.rs b/rust/operator-binary/src/catalog/postgresql.rs index cb052a759..fd4399f4c 100644 --- a/rust/operator-binary/src/catalog/postgresql.rs +++ b/rust/operator-binary/src/catalog/postgresql.rs @@ -22,8 +22,11 @@ impl ToCatalogConfig for PostgresqlConnector { _trino_version: u16, ) -> Result { let mut config = CatalogConfig::new(catalog_name.to_string(), CONNECTOR_NAME); - // SAFETY: `unique_database_name` must only contains ASCII letters and underscores. - let unique_database_name = format!("POSTGRESQL_{}", catalog_name.replace('-', "_")); + // SAFETY: `unique_database_name` must only contains uppercase ASCII letters and underscores. + let unique_database_name = format!( + "POSTGRESQL_{}", + catalog_name.replace('-', "_").to_uppercase() + ); let jdbc_connection_details = self .inner .jdbc_connection_details(&unique_database_name) From 2f5ee44a87d1b355f72a1e6016abffe1d7371edc Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 27 Apr 2026 14:09:31 +0200 Subject: [PATCH 4/4] typo --- rust/operator-binary/src/catalog/postgresql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/catalog/postgresql.rs b/rust/operator-binary/src/catalog/postgresql.rs index fd4399f4c..e5731641b 100644 --- a/rust/operator-binary/src/catalog/postgresql.rs +++ b/rust/operator-binary/src/catalog/postgresql.rs @@ -22,7 +22,7 @@ impl ToCatalogConfig for PostgresqlConnector { _trino_version: u16, ) -> Result { let mut config = CatalogConfig::new(catalog_name.to_string(), CONNECTOR_NAME); - // SAFETY: `unique_database_name` must only contains uppercase ASCII letters and underscores. + // SAFETY: `unique_database_name` must only contain uppercase ASCII letters and underscores. let unique_database_name = format!( "POSTGRESQL_{}", catalog_name.replace('-', "_").to_uppercase()