From 1823648d0434c1e98b6a27d08f5b371069fe16cc Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 21 Apr 2026 13:29:49 +0200 Subject: [PATCH 01/11] implement changes according to airflow pattern, where appropriate --- Cargo.lock | 20 +++--- Cargo.nix | 22 +++--- Cargo.toml | 2 +- crate-hashes.json | 18 ++--- deploy/config-spec/properties.yaml | 7 +- .../superset-operator/configs/properties.yaml | 7 +- .../getting_started/superset-credentials.yaml | 4 +- .../superset-load-examples-job.yaml | 20 ++++-- .../superset-load-examples-job.yaml.j2 | 20 ++++-- .../examples/getting_started/superset.yaml | 15 ++++- .../pages/getting_started/first_steps.adoc | 12 ++-- .../usage-guide/database-connections.adoc | 35 ++++++++++ .../superset/pages/usage-guide/index.adoc | 4 +- docs/modules/superset/partials/nav.adoc | 1 + extra/crds.yaml | 65 +++++++++++++++++- rust/operator-binary/src/crd/affinity.rs | 11 ++- rust/operator-binary/src/crd/databases.rs | 31 +++++++++ rust/operator-binary/src/crd/mod.rs | 25 ++++--- .../src/druid_connection_controller.rs | 57 ++++++++++++---- .../src/superset_controller.rs | 67 +++++++++++++++---- .../20-install-superset.yaml.j2 | 19 ++++-- .../30-install-superset-pause.yaml.j2 | 37 +--------- .../40-install-superset-stop.yaml.j2 | 37 +--------- .../50-install-superset-restart.yaml.j2 | 37 +--------- .../20-install-superset.yaml.j2 | 19 ++++-- .../external-access/install-superset.yaml.j2 | 19 ++++-- .../kuttl/ldap/50-install-superset.yaml.j2 | 19 ++++-- .../kuttl/logging/21-install-superset.yaml.j2 | 19 ++++-- .../kuttl/oidc/40_install-superset.yaml.j2 | 19 ++++-- tests/templates/kuttl/opa/40_superset.yaml.j2 | 19 ++++-- .../resources/20-install-superset.yaml.j2 | 19 ++++-- .../kuttl/smoke/30-install-superset.yaml.j2 | 19 ++++-- 32 files changed, 482 insertions(+), 243 deletions(-) create mode 100644 docs/modules/superset/pages/usage-guide/database-connections.adoc create mode 100644 rust/operator-binary/src/crd/databases.rs diff --git a/Cargo.lock b/Cargo.lock index 58c12b12..7829ffbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "darling", "regex", @@ -2881,7 +2881,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "const-oid", "ecdsa", @@ -2904,8 +2904,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.110.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +version = "0.110.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "base64", "clap", @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "darling", "proc-macro2", @@ -2957,7 +2957,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "jiff", "k8s-openapi", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "axum", "clap", @@ -3021,7 +3021,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.9.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "schemars", "serde", @@ -3034,7 +3034,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.9.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "convert_case", "convert_case_extras", @@ -3052,7 +3052,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#95490f2d703d20cf1895e5976fdc5fb8f02aa293" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 3290339a..b0d9648f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4862,7 +4862,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "k8s_version"; @@ -9491,7 +9491,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "stackable_certs"; @@ -9589,12 +9589,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.110.0"; + version = "0.110.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "stackable_operator"; @@ -9774,7 +9774,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; procMacro = true; @@ -9809,7 +9809,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "stackable_shared"; @@ -9944,7 +9944,7 @@ rec { { name = "stackable-operator"; packageId = "stackable-operator"; - features = [ "webhook" ]; + features = [ "crds" "webhook" ]; } { name = "strum"; @@ -9996,7 +9996,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "stackable_telemetry"; @@ -10106,7 +10106,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "stackable_versioned"; @@ -10150,7 +10150,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; procMacro = true; @@ -10218,7 +10218,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; + rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; }; libName = "stackable_webhook"; diff --git a/Cargo.toml b/Cargo.toml index acb3ff10..22382be4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/superset-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.110.0", features = ["webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.110.1", features = ["crds", "webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 33e75762..92bf1f24 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#k8s-version@0.1.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-certs@0.4.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator-derive@0.3.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator@0.110.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-shared@0.1.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-telemetry@0.6.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned-macros@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-webhook@0.9.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#k8s-version@0.1.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-certs@0.4.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator-derive@0.3.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator@0.110.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-shared@0.1.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-telemetry@0.6.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned-macros@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-webhook@0.9.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index 35ac7ca1..de0eee44 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,10 +1,11 @@ +--- version: 0.1.0 spec: units: [] properties: - - property: &credentialsSecret + - property: &credentialsSecretName propertyNames: - - name: "credentialsSecret" + - name: "credentialsSecretName" kind: type: "env" datatype: @@ -13,7 +14,7 @@ properties: - name: "node" required: true asOfVersion: "0.0.0" - description: "The secret where the Superset credentials are stored." + description: "The name of the Secret where the Superset credentials are stored." - property: &rowLimit propertyNames: - name: "ROW_LIMIT" diff --git a/deploy/helm/superset-operator/configs/properties.yaml b/deploy/helm/superset-operator/configs/properties.yaml index 35ac7ca1..de0eee44 100644 --- a/deploy/helm/superset-operator/configs/properties.yaml +++ b/deploy/helm/superset-operator/configs/properties.yaml @@ -1,10 +1,11 @@ +--- version: 0.1.0 spec: units: [] properties: - - property: &credentialsSecret + - property: &credentialsSecretName propertyNames: - - name: "credentialsSecret" + - name: "credentialsSecretName" kind: type: "env" datatype: @@ -13,7 +14,7 @@ properties: - name: "node" required: true asOfVersion: "0.0.0" - description: "The secret where the Superset credentials are stored." + description: "The name of the Secret where the Superset credentials are stored." - property: &rowLimit propertyNames: - name: "ROW_LIMIT" diff --git a/docs/modules/superset/examples/getting_started/superset-credentials.yaml b/docs/modules/superset/examples/getting_started/superset-credentials.yaml index c7788b3a..49582af5 100644 --- a/docs/modules/superset/examples/getting_started/superset-credentials.yaml +++ b/docs/modules/superset/examples/getting_started/superset-credentials.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: simple-superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -10,5 +10,3 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql.default.svc.cluster.local/superset diff --git a/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml b/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml index a00960cb..4e46ddc7 100644 --- a/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml +++ b/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml @@ -15,9 +15,10 @@ spec: - name: superset image: oci.stackable.tech/sdp/superset:6.0.0-stackable0.0.0-dev command: [ - "/bin/sh", + "/bin/bash", "-c", - "mkdir --parents /stackable/app/pythonpath && \ + "export SQLALCHEMY_DATABASE_URI=\"postgresql://${DB_USERNAME}:${DB_PASSWORD}@superset-postgresql.default.svc.cluster.local/superset\" && \ + mkdir --parents /stackable/app/pythonpath && \ cp /stackable/config/* /stackable/app/pythonpath && \ echo 'SQLALCHEMY_EXAMPLES_URI = os.environ.get(\"SQLALCHEMY_DATABASE_URI\")' >> /stackable/app/pythonpath/superset_config.py && \ superset load_examples" @@ -26,13 +27,18 @@ spec: - name: SECRET_KEY valueFrom: secretKeyRef: - key: connections.secretKey - name: simple-superset-credentials - - name: SQLALCHEMY_DATABASE_URI + key: SECRET_KEY + name: simple-superset-secret-key + - name: DB_USERNAME valueFrom: secretKeyRef: - key: connections.sqlalchemyDatabaseUri - name: simple-superset-credentials + key: username + name: superset-postgresql-credentials + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: superset-postgresql-credentials volumeMounts: - mountPath: /stackable/config name: config diff --git a/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml.j2 b/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml.j2 index 67b6aba8..abb4aa0a 100644 --- a/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml.j2 +++ b/docs/modules/superset/examples/getting_started/superset-load-examples-job.yaml.j2 @@ -15,9 +15,10 @@ spec: - name: superset image: oci.stackable.tech/sdp/superset:6.0.0-stackable{{ versions.superset }} command: [ - "/bin/sh", + "/bin/bash", "-c", - "mkdir --parents /stackable/app/pythonpath && \ + "export SQLALCHEMY_DATABASE_URI=\"postgresql://${DB_USERNAME}:${DB_PASSWORD}@superset-postgresql.default.svc.cluster.local/superset\" && \ + mkdir --parents /stackable/app/pythonpath && \ cp /stackable/config/* /stackable/app/pythonpath && \ echo 'SQLALCHEMY_EXAMPLES_URI = os.environ.get(\"SQLALCHEMY_DATABASE_URI\")' >> /stackable/app/pythonpath/superset_config.py && \ superset load_examples" @@ -26,13 +27,18 @@ spec: - name: SECRET_KEY valueFrom: secretKeyRef: - key: connections.secretKey - name: simple-superset-credentials - - name: SQLALCHEMY_DATABASE_URI + key: SECRET_KEY + name: simple-superset-secret-key + - name: DB_USERNAME valueFrom: secretKeyRef: - key: connections.sqlalchemyDatabaseUri - name: simple-superset-credentials + key: username + name: superset-postgresql-credentials + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: superset-postgresql-credentials volumeMounts: - mountPath: /stackable/config name: config diff --git a/docs/modules/superset/examples/getting_started/superset.yaml b/docs/modules/superset/examples/getting_started/superset.yaml index 025997e7..3e9f6bd1 100644 --- a/docs/modules/superset/examples/getting_started/superset.yaml +++ b/docs/modules/superset/examples/getting_started/superset.yaml @@ -1,4 +1,12 @@ --- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset +--- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster metadata: @@ -7,7 +15,12 @@ spec: image: productVersion: 6.0.0 clusterConfig: - credentialsSecret: simple-superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials nodes: roleConfig: listenerClass: external-unstable diff --git a/docs/modules/superset/pages/getting_started/first_steps.adoc b/docs/modules/superset/pages/getting_started/first_steps.adoc index 4ff30978..cde7d19e 100644 --- a/docs/modules/superset/pages/getting_started/first_steps.adoc +++ b/docs/modules/superset/pages/getting_started/first_steps.adoc @@ -16,7 +16,7 @@ Refer to the xref:required-external-components[supported databases] for producti == Secret with Superset credentials -You need to create a secret with the required credentials (database connection credentials and an admin account for Superset). +You need to create secrets with the required credentials (an admin account for Superset and database connection credentials). Create a file called `superset-credentials.yaml`: [source,yaml] @@ -27,14 +27,12 @@ Apply the Secret: [source,bash] include::example$getting_started/getting_started.sh[tag=apply-superset-credentials] -The `connections.secretKey` will be used for securely signing the session cookies and can be used by the extensions for any other security-related needs. -It should be a long, random string. - -`connections.sqlalchemyDatabaseUri` must contain the connection string to the SQL database storing the Superset metadata. - The `adminUser` fields are used to create an admin user. If using non-default authentication (e.g., LDAP), the admin user is disabled. +The database connection credentials are stored in a separate secret. +Create it in the `superset.yaml` file (shown below) or apply it separately. + == Creation of a Superset node A Superset node must be created as a custom resource, create a file called `superset.yaml`: @@ -50,7 +48,7 @@ include::example$getting_started/getting_started.sh[tag=apply-superset-cluster] `metadata.name` contains the name of the Superset cluster. -The previously created secret must be referenced in `spec.clusterConfig.credentialsSecret`. +The previously created credentials secret must be referenced in `spec.clusterConfig.credentialsSecretName`, and the database connection is configured via `spec.clusterConfig.metadataDatabase`. The `rowLimit` configuration option defines the row limit when requesting chart data. diff --git a/docs/modules/superset/pages/usage-guide/database-connections.adoc b/docs/modules/superset/pages/usage-guide/database-connections.adoc new file mode 100644 index 00000000..29589401 --- /dev/null +++ b/docs/modules/superset/pages/usage-guide/database-connections.adoc @@ -0,0 +1,35 @@ += Database connections +:description: Configure Superset database connectivity. + +Superset requires a metadata database for storing slices, connections, tables, dashboards and other metadata. +The actual connection string is calculated by the operator so that the user does not need to remember the exact structure. + +== Typed connections + +[source,yaml] +---- +spec: + clusterConfig: + metadataDatabase: + postgresql: # <1> + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials # <2> +---- +<1> A reference to one of the supported database backends (e.g. `postgresql`). +<2> A reference to a Secret which must contain the two fields `username` and `password`. + +== Generic connections + +Alternatively, these connections can also be defined in full in a referenced secret: + +[source,yaml] +---- +spec: + clusterConfig: + metadataDatabase: + generic: + connectionUrlSecretName: superset-metadata # <1> +---- + +<1> A reference to a secret which must contain the single field `connectionUrl` e.g. `postgresql://superset:superset@superset-postgresql/superset` diff --git a/docs/modules/superset/pages/usage-guide/index.adoc b/docs/modules/superset/pages/usage-guide/index.adoc index 8d5d8be6..43febdd4 100644 --- a/docs/modules/superset/pages/usage-guide/index.adoc +++ b/docs/modules/superset/pages/usage-guide/index.adoc @@ -1,9 +1,9 @@ = Usage guide :page-aliases: usage.doc -:description: Superset usage guide: configure resources, connect to Apache Druid, set up authentication, logging, and monitoring. +:description: Superset usage guide: configure database connections, resources, connect to Apache Druid, set up authentication, logging, and monitoring. The usage guide covers various aspects of configuring Superset and interconnection with other tools. -Learn about defining the amount of xref:usage-guide/storage-resource-configuration.adoc[resources] Superset uses or learn how to xref:usage-guide/connecting-druid.adoc[connect to Apache Druid] operated by the xref:druid:index.adoc[]. +Learn how to configure the xref:usage-guide/database-connections.adoc[metadata database connection], define the amount of xref:usage-guide/storage-resource-configuration.adoc[resources] Superset uses, or learn how to xref:usage-guide/connecting-druid.adoc[connect to Apache Druid] operated by the xref:druid:index.adoc[]. Configure xref:usage-guide/security.adoc#authentication[authentication] with LDAP and xref:usage-guide/security.adoc#authorization[authorization]. Observe your Superset instance with xref:usage-guide/logging.adoc[log aggregation] and xref:usage-guide/monitoring.adoc[monitoring]. diff --git a/docs/modules/superset/partials/nav.adoc b/docs/modules/superset/partials/nav.adoc index 4b419323..904ff1e7 100644 --- a/docs/modules/superset/partials/nav.adoc +++ b/docs/modules/superset/partials/nav.adoc @@ -5,6 +5,7 @@ * xref:superset:usage-guide/index.adoc[] ** xref:superset:usage-guide/listenerclass.adoc[] ** xref:superset:usage-guide/storage-resource-configuration.adoc[] +** xref:superset:usage-guide/database-connections.adoc[] ** xref:superset:usage-guide/security.adoc[] ** xref:superset:usage-guide/connecting-druid.adoc[] ** xref:superset:usage-guide/monitoring.adoc[] diff --git a/extra/crds.yaml b/extra/crds.yaml index 9e53f171..ed2cfbe5 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -171,9 +171,9 @@ spec: and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. type: boolean type: object - credentialsSecret: + credentialsSecretName: description: |- - The name of the Secret object containing the admin user credentials and database connection details. + The name of the Secret object containing the admin user credentials. Read the [getting started guide first steps](https://docs.stackable.tech/home/nightly/superset/getting_started/first_steps) to find out more. @@ -186,6 +186,64 @@ spec: The token should be in the JWT format. nullable: true type: string + metadataDatabase: + description: Configure the database where Superset stores all its internal metadata. + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + description: |- + A generic SQLAlchemy database connection for database types not covered by a dedicated variant. + + Use this when you need to connect to a SQLAlchemy-compatible database that does not have a + first-class connection type. The complete connection URL is read from a Secret, giving the user + full control over the connection string including any driver-specific options. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete SQLAlchemy URL. + type: string + required: + - connectionUrlSecretName + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + type: object vectorAggregatorConfigMapName: description: |- Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). @@ -195,7 +253,8 @@ spec: nullable: true type: string required: - - credentialsSecret + - credentialsSecretName + - metadataDatabase type: object image: anyOf: diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 1d6b839d..2f7afef9 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -45,14 +45,21 @@ mod tests { image: productVersion: 4.1.4 clusterConfig: - credentialsSecret: superset-db-credentials + credentialsSecretName: superset-db-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials nodes: roleGroups: default: replicas: 1 "#; + let deserializer = serde_yaml::Deserializer::from_str(input); let superset: v1alpha1::SupersetCluster = - serde_yaml::from_str(input).expect("illegal test input"); + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) + .expect("illegal test input"); let merged_config = superset .merged_config(&SupersetRole::Node, &superset.node_rolegroup_ref("default")) .unwrap(); diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs new file mode 100644 index 00000000..5862882c --- /dev/null +++ b/rust/operator-binary/src/crd/databases.rs @@ -0,0 +1,31 @@ +use std::ops::Deref; + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + database_connections::{ + databases::postgresql::PostgresqlConnection, + drivers::sqlalchemy::{GenericSqlAlchemyDatabaseConnection, SqlAlchemyDatabaseConnection}, + }, + schemars::{self, JsonSchema}, +}; + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MetadataDatabaseConnection { + // Docs are on the struct + Postgresql(PostgresqlConnection), + + // Docs are on the struct + Generic(GenericSqlAlchemyDatabaseConnection), +} + +impl Deref for MetadataDatabaseConnection { + type Target = dyn SqlAlchemyDatabaseConnection; + + fn deref(&self) -> &Self::Target { + match self { + Self::Postgresql(p) => p, + Self::Generic(g) => g, + } + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index dc058d40..fddb7f56 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -34,10 +34,14 @@ use stackable_operator::{ }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use crate::{crd::v1alpha1::SupersetRoleConfig, listener::default_listener_class}; +use crate::{ + crd::{databases::MetadataDatabaseConnection, v1alpha1::SupersetRoleConfig}, + listener::default_listener_class, +}; pub mod affinity; pub mod authentication; +pub mod databases; pub mod druidconnection; pub const FIELD_MANAGER: &str = "superset-operator"; @@ -52,6 +56,8 @@ pub const MAX_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { unit: BinaryMultiple::Mebi, }; +pub const INTERNAL_SECRET_SECRET_KEY: &str = "SECRET_KEY"; + pub const APP_PORT_NAME: &str = "http"; pub const APP_PORT: u16 = 8088; pub const METRICS_PORT_NAME: &str = "metrics"; @@ -199,11 +205,14 @@ pub mod versioned { #[serde(skip_serializing_if = "Option::is_none")] pub authorization: Option, - /// The name of the Secret object containing the admin user credentials and database connection details. + /// Configure the database where Superset stores all its internal metadata. + pub metadata_database: MetadataDatabaseConnection, + + /// The name of the Secret object containing the admin user credentials. /// Read the /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/superset/getting_started/first_steps) /// to find out more. - pub credentials_secret: String, + pub credentials_secret_name: String, /// Cluster operations like pause reconciliation or cluster stop. #[serde(default)] @@ -435,7 +444,6 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { } impl v1alpha1::SupersetConfig { - pub const CREDENTIALS_SECRET_PROPERTY: &'static str = "credentialsSecret"; pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; fn default_config(cluster_name: &str, role: &SupersetRole) -> v1alpha1::SupersetConfigFragment { @@ -469,17 +477,12 @@ impl Configuration for v1alpha1::SupersetConfigFragment { _role_name: &str, ) -> Result>, product_config_utils::Error> { let mut result = BTreeMap::new(); - result.insert( - v1alpha1::SupersetConfig::CREDENTIALS_SECRET_PROPERTY.to_string(), - Some(cluster.spec.cluster_config.credentials_secret.clone()), - ); if let Some(msec) = &cluster.spec.cluster_config.mapbox_secret { result.insert( v1alpha1::SupersetConfig::MAPBOX_SECRET_PROPERTY.to_string(), Some(msec.clone()), ); } - Ok(result) } @@ -521,6 +524,10 @@ impl HasStatusCondition for v1alpha1::SupersetCluster { } impl v1alpha1::SupersetCluster { + pub fn shared_secret_key_secret_name(&self) -> String { + format!("{}-secret-key", &self.name_any()) + } + /// The name of the group-listener provided for a specific role. /// Nodes will use this group listener so that only one load balancer /// is needed for that role. diff --git a/rust/operator-binary/src/druid_connection_controller.rs b/rust/operator-binary/src/druid_connection_controller.rs index 26322a3f..81fc84e2 100644 --- a/rust/operator-binary/src/druid_connection_controller.rs +++ b/rust/operator-binary/src/druid_connection_controller.rs @@ -9,6 +9,7 @@ use stackable_operator::{ }, client::Client, commons::product_image_selection::{self, ResolvedProductImage}, + database_connections::TemplatingMechanism, k8s_openapi::api::{ batch::v1::{Job, JobSpec}, core::v1::{ConfigMap, PodSpec, PodTemplateSpec}, @@ -26,7 +27,9 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ APP_NAME, OPERATOR_NAME, - crd::{PYTHONPATH, SUPERSET_CONFIG_FILENAME, druidconnection, v1alpha1}, + crd::{ + INTERNAL_SECRET_SECRET_KEY, PYTHONPATH, SUPERSET_CONFIG_FILENAME, druidconnection, v1alpha1, + }, rbac, superset_controller::DOCKER_IMAGE_BASE_NAME, util::{JobState, get_job_state}, @@ -315,7 +318,7 @@ async fn build_import_job( ) -> Result { let mut commands = vec![]; - let config = "import os; SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URI')"; + let config = "import os; SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')"; commands.push(format!("mkdir -p {PYTHONPATH}")); commands.push(format!( "echo \"{config}\" > {PYTHONPATH}/{SUPERSET_CONFIG_FILENAME}" @@ -327,18 +330,48 @@ async fn build_import_job( "superset import_datasources -p /tmp/druids.yaml", )); - let secret = &superset_cluster.spec.cluster_config.credentials_secret; + // "METADATA" is the prefix for the env vars that hold the database credentials + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It must match + // the prefix used by the airflow-operator for consistency. + let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; + let metadata_database_connection_details = superset_cluster + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); - let container = ContainerBuilder::new("superset-import-druid-connection") - .expect("ContainerBuilder not created") + let mut container_builder = ContainerBuilder::new("superset-import-druid-connection") + .expect("ContainerBuilder not created"); + container_builder .image_from_product_image(resolved_product_image) - .command(vec!["/bin/sh".to_string()]) - .args(vec![String::from("-c"), commands.join("; ")]) - .add_env_var_from_secret("DATABASE_URI", secret, "connections.sqlalchemyDatabaseUri") - // From 2.1.0 superset barfs if the SECRET_KEY is not set properly. This causes the import job to fail. - // Setting the env var is enough to be picked up: https://superset.apache.org/docs/installation/configuring-superset/#configuration - .add_env_var_from_secret("SUPERSET_SECRET_KEY", secret, "connections.secretKey") - .build(); + .command(vec!["/bin/bash".to_string()]) + .args(vec![ + String::from("-c"), + // SQLALCHEMY_DATABASE_URI contains a template with bash variable references + // (e.g. ${METADATA_DATABASE_USERNAME}) that must be resolved before Python + // reads it via os.environ. Airflow's config system calls expandvars() + // on env var values automatically (see [1]), but Superset does not - so we + // resolve them here with eval. + // [1] https://github.com/apache/airflow/blob/2.10.5/airflow/configuration.py#L1084-L1086 + // (same behaviour in 3.x: shared/configuration/src/airflow_shared/configuration/parser.py) + format!( + "export SQLALCHEMY_DATABASE_URI=$(eval echo \"$SQLALCHEMY_DATABASE_URI\")\n{}", + commands.join("; ") + ), + ]) + .add_env_var( + "SQLALCHEMY_DATABASE_URI", + metadata_database_connection_details.url_template.clone(), + ) + .add_env_var_from_secret( + "SUPERSET_SECRET_KEY", + superset_cluster.shared_secret_key_secret_name(), + INTERNAL_SECRET_SECRET_KEY, + ); + + metadata_database_connection_details.add_to_container(&mut container_builder); + + let container = container_builder.build(); let pod = PodTemplateSpec { metadata: Some( diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 6be611fb..0a1b013c 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -34,9 +34,11 @@ use stackable_operator::{ cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ product_image_selection::{self, ResolvedProductImage}, + random_secret_creation, rbac::build_rbac_resources, }, crd::authentication::oidc, + database_connections::TemplatingMechanism, k8s_openapi::{ DeepMerge, api::{ @@ -80,9 +82,9 @@ use crate::{ config::{self, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ - APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, - SupersetConfigOptions, SupersetRole, + APP_NAME, APP_PORT, INTERNAL_SECRET_SECRET_KEY, METRICS_PORT, METRICS_PORT_NAME, + PYTHONPATH, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, + SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, SupersetRole, authentication::{ SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, }, @@ -305,6 +307,11 @@ pub enum Error { ResolveProductImage { source: product_image_selection::Error, }, + + #[snafu(display("failed to create SECRET_KEY secret"))] + InvalidSecretKey { + source: random_secret_creation::Error, + }, } type Result = std::result::Result; @@ -410,6 +417,16 @@ pub async fn reconcile_superset( .await .context(ApplyRoleBindingSnafu)?; + random_secret_creation::create_random_secret_if_not_exists( + &superset.shared_secret_key_secret_name(), + INTERNAL_SECRET_SECRET_KEY, + 256, + superset, + client, + ) + .await + .context(InvalidSecretKeySnafu)?; + let mut ss_cond_builder = StatefulSetConditionBuilder::default(); for (rolegroup_name, rolegroup_config) in role_node_config.iter() { @@ -701,6 +718,16 @@ fn build_server_rolegroup_statefulset( .affinity(&merged_config.affinity) .service_account_name(sa_name); + // "METADATA" is the prefix for the env vars that hold the database credentials + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It must match + // the prefix used by the airflow-operator for consistency. + let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; + let metadata_database_connection_details = superset + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) .context(InvalidContainerNameSnafu)?; @@ -709,14 +736,7 @@ fn build_server_rolegroup_statefulset( .cloned() .unwrap_or_default() { - if name == SupersetConfig::CREDENTIALS_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret("SECRET_KEY", &value, "connections.secretKey"); - superset_cb.add_env_var_from_secret( - "SQLALCHEMY_DATABASE_URI", - &value, - "connections.sqlalchemyDatabaseUri", - ); - } else if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { superset_cb.add_env_var_from_secret( "MAPBOX_API_KEY", &value, @@ -727,6 +747,19 @@ fn build_server_rolegroup_statefulset( }; } + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + INTERNAL_SECRET_SECRET_KEY, + ); + + // Database connection URL from metadataDatabase + superset_cb.add_env_var( + "SQLALCHEMY_DATABASE_URI", + metadata_database_connection_details.url_template.clone(), + ); + add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; let webserver_timeout = node_config @@ -737,7 +770,7 @@ fn build_server_rolegroup_statefulset( .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; - let secret = &superset.spec.cluster_config.credentials_secret; + let secret = &superset.spec.cluster_config.credentials_secret_name; superset_cb .image_from_product_image(resolved_product_image) @@ -770,6 +803,14 @@ fn build_server_rolegroup_statefulset( {auth_commands} + # SQLALCHEMY_DATABASE_URI contains a template with bash variable references + # (e.g. ${{METADATA_DATABASE_USERNAME}}) that must be resolved before Python + # reads it via os.environ.get(). Airflow's config system calls expandvars() + # on env var values automatically (see [1]), but Superset does not - so we + # resolve them here with eval. + # [1] https://github.com/apache/airflow/blob/2.10.5/airflow/configuration.py#L1084-L1086 + # (same behaviour in 3.x: shared/configuration/src/airflow_shared/configuration/parser.py) + export SQLALCHEMY_DATABASE_URI=$(eval echo \"$SQLALCHEMY_DATABASE_URI\") superset db upgrade set +x echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' @@ -813,6 +854,8 @@ fn build_server_rolegroup_statefulset( .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)?; + metadata_database_connection_details.add_to_container(&mut superset_cb); + pb.add_container(superset_cb.build()); add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; diff --git a/tests/templates/kuttl/cluster-operation/20-install-superset.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-install-superset.yaml.j2 index e303157d..60283956 100644 --- a/tests/templates/kuttl/cluster-operation/20-install-superset.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-install-superset.yaml.j2 @@ -8,7 +8,7 @@ timeout: 300 apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -16,8 +16,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -33,7 +39,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 index 21d71995..9f1d643f 100644 --- a/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 @@ -2,48 +2,15 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep metadata: - name: install-superset + name: pause-superset timeout: 300 --- -apiVersion: v1 -kind: Secret -metadata: - name: superset-credentials -type: Opaque -stringData: - adminUser.username: admin - adminUser.firstname: Superset - adminUser.lastname: Admin - adminUser.email: admin@superset.com - adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset ---- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster metadata: name: superset spec: - image: -{% if test_scenario['values']['superset-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['superset-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['superset-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['superset-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent clusterConfig: clusterOperation: - stopped: false reconciliationPaused: true - credentialsSecret: superset-credentials -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - nodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 2 + stopped: false diff --git a/tests/templates/kuttl/cluster-operation/40-install-superset-stop.yaml.j2 b/tests/templates/kuttl/cluster-operation/40-install-superset-stop.yaml.j2 index 64e7dd92..ceba0759 100644 --- a/tests/templates/kuttl/cluster-operation/40-install-superset-stop.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/40-install-superset-stop.yaml.j2 @@ -2,48 +2,15 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep metadata: - name: install-superset + name: stop-superset timeout: 300 --- -apiVersion: v1 -kind: Secret -metadata: - name: superset-credentials -type: Opaque -stringData: - adminUser.username: admin - adminUser.firstname: Superset - adminUser.lastname: Admin - adminUser.email: admin@superset.com - adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset ---- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster metadata: name: superset spec: - image: -{% if test_scenario['values']['superset-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['superset-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['superset-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['superset-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent clusterConfig: clusterOperation: - stopped: true reconciliationPaused: false - credentialsSecret: superset-credentials -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - nodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 2 + stopped: true diff --git a/tests/templates/kuttl/cluster-operation/50-install-superset-restart.yaml.j2 b/tests/templates/kuttl/cluster-operation/50-install-superset-restart.yaml.j2 index 43f8c936..4bb858d8 100644 --- a/tests/templates/kuttl/cluster-operation/50-install-superset-restart.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/50-install-superset-restart.yaml.j2 @@ -2,48 +2,15 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep metadata: - name: install-superset + name: restart-superset timeout: 300 --- -apiVersion: v1 -kind: Secret -metadata: - name: superset-credentials -type: Opaque -stringData: - adminUser.username: admin - adminUser.firstname: Superset - adminUser.lastname: Admin - adminUser.email: admin@superset.com - adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset ---- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster metadata: name: superset spec: - image: -{% if test_scenario['values']['superset-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['superset-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['superset-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['superset-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent clusterConfig: clusterOperation: - stopped: false reconciliationPaused: false - credentialsSecret: superset-credentials -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - nodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 2 + stopped: false diff --git a/tests/templates/kuttl/druid-connection/20-install-superset.yaml.j2 b/tests/templates/kuttl/druid-connection/20-install-superset.yaml.j2 index 9d8b4c66..b935f069 100644 --- a/tests/templates/kuttl/druid-connection/20-install-superset.yaml.j2 +++ b/tests/templates/kuttl/druid-connection/20-install-superset.yaml.j2 @@ -8,7 +8,7 @@ timeout: 300 apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -16,8 +16,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -33,7 +39,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/external-access/install-superset.yaml.j2 b/tests/templates/kuttl/external-access/install-superset.yaml.j2 index 4bd53e65..51097bcb 100644 --- a/tests/templates/kuttl/external-access/install-superset.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-superset.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -10,8 +10,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -27,7 +33,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials nodes: roleConfig: listenerClass: test-external-unstable-$NAMESPACE diff --git a/tests/templates/kuttl/ldap/50-install-superset.yaml.j2 b/tests/templates/kuttl/ldap/50-install-superset.yaml.j2 index 5835a528..707e70b4 100644 --- a/tests/templates/kuttl/ldap/50-install-superset.yaml.j2 +++ b/tests/templates/kuttl/ldap/50-install-superset.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: superset-with-ldap-credentials + name: superset-with-ldap-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -10,8 +10,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: v1 kind: Secret @@ -47,7 +53,12 @@ spec: {%- endif %} userRegistrationRole: Admin - credentialsSecret: superset-with-ldap-credentials + credentialsSecretName: superset-with-ldap-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/logging/21-install-superset.yaml.j2 b/tests/templates/kuttl/logging/21-install-superset.yaml.j2 index 029cd73d..c0124276 100644 --- a/tests/templates/kuttl/logging/21-install-superset.yaml.j2 +++ b/tests/templates/kuttl/logging/21-install-superset.yaml.j2 @@ -8,7 +8,7 @@ timeout: 300 apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -16,8 +16,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: v1 kind: ConfigMap @@ -61,7 +67,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials vectorAggregatorConfigMapName: superset-vector-aggregator-discovery nodes: roleGroups: diff --git a/tests/templates/kuttl/oidc/40_install-superset.yaml.j2 b/tests/templates/kuttl/oidc/40_install-superset.yaml.j2 index b2c5e3e3..3c6a2e67 100644 --- a/tests/templates/kuttl/oidc/40_install-superset.yaml.j2 +++ b/tests/templates/kuttl/oidc/40_install-superset.yaml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -11,8 +11,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: aQC11KVUJ3yTVcy2 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: v1 kind: Secret @@ -51,7 +57,12 @@ spec: - authenticationClass: keycloak2-$NAMESPACE oidc: clientCredentialsSecret: superset-keycloak2-client - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index 3d6a5d15..3ecd81b3 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -11,8 +11,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: aQC11KVUJ3yTVcy2 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -32,7 +38,12 @@ spec: roleMappingFromOpa: configMapName: opa package: superset - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/resources/20-install-superset.yaml.j2 b/tests/templates/kuttl/resources/20-install-superset.yaml.j2 index 99a5bbee..98e226ea 100644 --- a/tests/templates/kuttl/resources/20-install-superset.yaml.j2 +++ b/tests/templates/kuttl/resources/20-install-superset.yaml.j2 @@ -6,7 +6,7 @@ timeout: 300 apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -14,8 +14,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -31,7 +37,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 index 019bae50..5f994661 100644 --- a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 @@ -8,7 +8,7 @@ timeout: 300 apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -16,8 +16,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -33,7 +39,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From 0521f1dc53fa2c3ec4bbb0dfb42a65b9e00459ca Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 21 Apr 2026 14:26:51 +0200 Subject: [PATCH 02/11] added credentialsSecretName back as it is still in product config --- rust/operator-binary/src/crd/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index fddb7f56..2a86f412 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -444,6 +444,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { } impl v1alpha1::SupersetConfig { + pub const CREDENTIALS_SECRET_NAME_PROPERTY: &'static str = "credentialsSecretName"; pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; fn default_config(cluster_name: &str, role: &SupersetRole) -> v1alpha1::SupersetConfigFragment { @@ -477,6 +478,10 @@ impl Configuration for v1alpha1::SupersetConfigFragment { _role_name: &str, ) -> Result>, product_config_utils::Error> { let mut result = BTreeMap::new(); + result.insert( + v1alpha1::SupersetConfig::CREDENTIALS_SECRET_NAME_PROPERTY.to_string(), + Some(cluster.spec.cluster_config.credentials_secret_name.clone()), + ); if let Some(msec) = &cluster.spec.cluster_config.mapbox_secret { result.insert( v1alpha1::SupersetConfig::MAPBOX_SECRET_PROPERTY.to_string(), From ae7d36840e232d9b8accfe0c9d85d27ce395bb26 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 21 Apr 2026 14:50:28 +0200 Subject: [PATCH 03/11] changelog, rbac --- CHANGELOG.md | 6 ++++++ .../superset-operator/templates/clusterrole-operator.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7aeea7..bc043509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,16 @@ - Bump `stackable-operator` to 0.110.0 and `kube` to 3.1.0 ([#719]). - Support setting `clientAuthenticationMethod` for OIDC authentication. The value is passed through to the Flask-AppBuilder config as `token_endpoint_auth_method` ([#719]). - BREAKING: Rename `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` in `superset_config.py` for arbitrary Python code to `FILE_HEADER` and `FILE_FOOTER` ([#719], [#721]). +- BREAKING: The `.clusterConfig.credentialsSecret` field has been renamed to `.clusterConfig.credentialsSecretName` for consistency ([#722]). +- BREAKING: Implement generic database connection. + This means you need to replace your simple database connection string with a typed struct. + This struct is consistent between different CRDs, so that you can easily copy/paste it between stacklets. + More information can be found in the [Superset database documentation](https://docs.stackable.tech/home/nightly/superset/usage-guide/database-connections) for details ([#722]). [#717]: https://github.com/stackabletech/superset-operator/pull/717 [#719]: https://github.com/stackabletech/superset-operator/pull/719 [#721]: https://github.com/stackabletech/superset-operator/pull/721 +[#722]: https://github.com/stackabletech/superset-operator/pull/722 ## [26.3.0] - 2026-03-16 diff --git a/deploy/helm/superset-operator/templates/clusterrole-operator.yaml b/deploy/helm/superset-operator/templates/clusterrole-operator.yaml index a81c1a2a..567dbabd 100644 --- a/deploy/helm/superset-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/superset-operator/templates/clusterrole-operator.yaml @@ -16,10 +16,12 @@ rules: # Manage core namespaced resources created per SupersetCluster. # All resources are applied via Server-Side Apply (create + patch) and tracked for # orphan cleanup (list + delete). ReconciliationPaused uses get. + # Secrets are needed for auto-generated SECRET_KEY (random_secret_creation). - apiGroups: - "" resources: - configmaps + - secrets - services verbs: - create From c9a4bd136d1af083b0a3e7a9cb70dc186c3c5ed1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 21 Apr 2026 15:26:13 +0200 Subject: [PATCH 04/11] add the replica change back in --- .../kuttl/cluster-operation/30-install-superset-pause.yaml.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 index 9f1d643f..1170805d 100644 --- a/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-install-superset-pause.yaml.j2 @@ -14,3 +14,7 @@ spec: clusterOperation: reconciliationPaused: true stopped: false + nodes: + roleGroups: + default: + replicas: 2 From 00524409f415c7d1d8dfa8063cc72c822d104298 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 22 Apr 2026 11:59:25 +0200 Subject: [PATCH 05/11] Apply suggestions from code review Co-authored-by: Sebastian Bernauer --- .../modules/superset/pages/getting_started/first_steps.adoc | 4 ++-- .../superset/pages/usage-guide/database-connections.adoc | 6 +++--- rust/operator-binary/src/druid_connection_controller.rs | 2 +- rust/operator-binary/src/superset_controller.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/modules/superset/pages/getting_started/first_steps.adoc b/docs/modules/superset/pages/getting_started/first_steps.adoc index cde7d19e..b3747def 100644 --- a/docs/modules/superset/pages/getting_started/first_steps.adoc +++ b/docs/modules/superset/pages/getting_started/first_steps.adoc @@ -16,7 +16,7 @@ Refer to the xref:required-external-components[supported databases] for producti == Secret with Superset credentials -You need to create secrets with the required credentials (an admin account for Superset and database connection credentials). +You need to create two Secrets with the required credentials (an admin account for Superset as well as database connection credentials). Create a file called `superset-credentials.yaml`: [source,yaml] @@ -48,7 +48,7 @@ include::example$getting_started/getting_started.sh[tag=apply-superset-cluster] `metadata.name` contains the name of the Superset cluster. -The previously created credentials secret must be referenced in `spec.clusterConfig.credentialsSecretName`, and the database connection is configured via `spec.clusterConfig.metadataDatabase`. +The previously created credentials Secret must be referenced in `spec.clusterConfig.credentialsSecretName`, and the database connection Secret is configured via `spec.clusterConfig.metadataDatabase`. The `rowLimit` configuration option defines the row limit when requesting chart data. diff --git a/docs/modules/superset/pages/usage-guide/database-connections.adoc b/docs/modules/superset/pages/usage-guide/database-connections.adoc index 29589401..355762fa 100644 --- a/docs/modules/superset/pages/usage-guide/database-connections.adoc +++ b/docs/modules/superset/pages/usage-guide/database-connections.adoc @@ -17,11 +17,11 @@ spec: credentialsSecretName: superset-postgresql-credentials # <2> ---- <1> A reference to one of the supported database backends (e.g. `postgresql`). -<2> A reference to a Secret which must contain the two fields `username` and `password`. +<2> A reference to a Secret which must contain the two keys `username` and `password`. == Generic connections -Alternatively, these connections can also be defined in full in a referenced secret: +Alternatively, these connections can also be defined in full in a referenced Secret: [source,yaml] ---- @@ -32,4 +32,4 @@ spec: connectionUrlSecretName: superset-metadata # <1> ---- -<1> A reference to a secret which must contain the single field `connectionUrl` e.g. `postgresql://superset:superset@superset-postgresql/superset` +<1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `postgresql://superset:superset@superset-postgresql/superset` diff --git a/rust/operator-binary/src/druid_connection_controller.rs b/rust/operator-binary/src/druid_connection_controller.rs index 81fc84e2..944830e5 100644 --- a/rust/operator-binary/src/druid_connection_controller.rs +++ b/rust/operator-binary/src/druid_connection_controller.rs @@ -331,7 +331,7 @@ async fn build_import_job( )); // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It must match + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match // the prefix used by the airflow-operator for consistency. let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; let metadata_database_connection_details = superset_cluster diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 0a1b013c..23c4beac 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -719,7 +719,7 @@ fn build_server_rolegroup_statefulset( .service_account_name(sa_name); // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It must match + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match // the prefix used by the airflow-operator for consistency. let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; let metadata_database_connection_details = superset From d3f7c5a836ffacbb225b04908cd25ac97682b26b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:07:33 +0200 Subject: [PATCH 06/11] Update rust/operator-binary/src/superset_controller.rs Co-authored-by: Sebastian Bernauer --- rust/operator-binary/src/superset_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 23c4beac..f9534a8c 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -309,7 +309,7 @@ pub enum Error { }, #[snafu(display("failed to create SECRET_KEY secret"))] - InvalidSecretKey { + CreateSecretKeySecret { source: random_secret_creation::Error, }, } From 1b4c7166db67fa332b2005a5c64c345f62bb661b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 22 Apr 2026 12:20:46 +0200 Subject: [PATCH 07/11] review changes --- Cargo.lock | 73 ++++++---- Cargo.nix | 137 +++++++++++++----- Cargo.toml | 2 +- rust/operator-binary/src/crd/affinity.rs | 2 +- .../src/superset_controller.rs | 2 +- 5 files changed, 152 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7829ffbe..b64cb9ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,9 +331,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -392,11 +392,12 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" dependencies = [ "const_format_proc_macros", + "konst", ] [[package]] @@ -1531,6 +1532,21 @@ dependencies = [ "snafu 0.9.0", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kube" version = "3.1.0" @@ -1773,7 +1789,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -2105,9 +2121,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] @@ -2219,9 +2235,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", @@ -2494,9 +2510,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -3114,6 +3130,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -3269,9 +3291,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", @@ -3464,11 +3486,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -3561,9 +3584,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "ucd-trie" @@ -3661,9 +3684,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ "wit-bindgen", ] @@ -3886,18 +3909,18 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen" -version = "0.51.0" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" diff --git a/Cargo.nix b/Cargo.nix index b0d9648f..1a40390e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1060,10 +1060,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.6.0"; + version = "4.6.1"; edition = "2024"; crateBin = []; - sha256 = "0l8k0ja5rf4hpn2g98bqv5m6lkh2q6b6likjpmm6fjw3cxdsz4xi"; + sha256 = "0lcf88l7vlg796rrqr7wipbbmfa5sgsgx4211b7xmxxv8dz13nqx"; dependencies = [ { name = "clap_builder"; @@ -1141,9 +1141,9 @@ rec { }; "clap_derive" = rec { crateName = "clap_derive"; - version = "4.6.0"; + version = "4.6.1"; edition = "2024"; - sha256 = "0snapc468s7n3avr33dky4y7rmb7ha3qsp9l0k5vh6jacf5bs40i"; + sha256 = "1acpz49hi00iv9jkapixjzcv7s51x8qkfaqscjm36rqgf428dkpj"; procMacro = true; dependencies = [ { @@ -1226,9 +1226,9 @@ rec { }; "const_format" = rec { crateName = "const_format"; - version = "0.2.35"; + version = "0.2.36"; edition = "2021"; - sha256 = "1b9h03z3k76ail1ldqxcqmsc4raa7dwgwwqwrjf6wmism5lp9akz"; + sha256 = "07ncczs8yndga2f8p4386c827l4fxwzl0pbwp7ijnhcsmlbsd0a4"; authors = [ "rodrimati1992 " ]; @@ -1237,6 +1237,12 @@ rec { name = "const_format_proc_macros"; packageId = "const_format_proc_macros"; } + { + name = "konst"; + packageId = "konst"; + usesDefaultFeatures = false; + features = [ "rust_1_64" ]; + } ]; features = { "__debug" = [ "const_format_proc_macros/debug" ]; @@ -1250,10 +1256,9 @@ rec { "constant_time_as_str" = [ "fmt" ]; "derive" = [ "fmt" "const_format_proc_macros/derive" ]; "fmt" = [ "rust_1_83" ]; - "konst" = [ "dep:konst" ]; "more_str_macros" = [ "rust_1_64" ]; "nightly_const_generics" = [ "const_generics" ]; - "rust_1_64" = [ "rust_1_51" "konst" "konst/rust_1_64" ]; + "rust_1_64" = [ "rust_1_51" ]; "rust_1_83" = [ "rust_1_64" ]; }; resolvedDefaultFeatures = [ "default" ]; @@ -4890,6 +4895,53 @@ rec { }; resolvedDefaultFeatures = [ "darling" ]; }; + "konst" = rec { + crateName = "konst"; + version = "0.2.20"; + edition = "2018"; + sha256 = "1yyf1fhk28wbf1lqrga9as4cpfmpbry9a5vvdqyxgz14g3nk708j"; + authors = [ + "rodrimati1992 " + ]; + dependencies = [ + { + name = "konst_macro_rules"; + packageId = "konst_macro_rules"; + } + ]; + features = { + "__ui" = [ "__test" "trybuild" "rust_latest_stable" ]; + "const_generics" = [ "rust_1_51" ]; + "constant_time_slice" = [ "rust_latest_stable" ]; + "default" = [ "cmp" "parsing" ]; + "deref_raw_in_fn" = [ "rust_1_56" ]; + "konst_proc_macros" = [ "dep:konst_proc_macros" ]; + "mut_refs" = [ "rust_latest_stable" "konst_macro_rules/mut_refs" ]; + "nightly_mut_refs" = [ "mut_refs" "konst_macro_rules/nightly_mut_refs" ]; + "parsing" = [ "parsing_no_proc" "konst_proc_macros" ]; + "rust_1_51" = [ "konst_macro_rules/rust_1_51" ]; + "rust_1_55" = [ "rust_1_51" "konst_macro_rules/rust_1_55" ]; + "rust_1_56" = [ "rust_1_55" "konst_macro_rules/rust_1_56" ]; + "rust_1_57" = [ "rust_1_56" "konst_macro_rules/rust_1_57" ]; + "rust_1_61" = [ "rust_1_57" "konst_macro_rules/rust_1_61" ]; + "rust_1_64" = [ "rust_1_61" ]; + "rust_latest_stable" = [ "rust_1_64" ]; + "trybuild" = [ "dep:trybuild" ]; + }; + resolvedDefaultFeatures = [ "rust_1_51" "rust_1_55" "rust_1_56" "rust_1_57" "rust_1_61" "rust_1_64" ]; + }; + "konst_macro_rules" = rec { + crateName = "konst_macro_rules"; + version = "0.2.19"; + edition = "2018"; + sha256 = "0dswja0dqcww4x3fwjnirc0azv2n6cazn8yv0kddksd8awzkz4x4"; + authors = [ + "rodrimati1992 " + ]; + features = { + }; + resolvedDefaultFeatures = [ "rust_1_51" "rust_1_55" "rust_1_56" "rust_1_57" "rust_1_61" ]; + }; "kube" = rec { crateName = "kube"; version = "3.1.0"; @@ -5843,7 +5895,7 @@ rec { } { name = "rand"; - packageId = "rand 0.8.5"; + packageId = "rand 0.8.6"; optional = true; usesDefaultFeatures = false; } @@ -5862,7 +5914,7 @@ rec { devDependencies = [ { name = "rand"; - packageId = "rand 0.8.5"; + packageId = "rand 0.8.6"; features = [ "small_rng" ]; } ]; @@ -6969,9 +7021,9 @@ rec { }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; - version = "0.2.6"; + version = "0.2.7"; edition = "2018"; - sha256 = "18wrsx7fjwc2kgbpfjfm3igv3vdzsidmjhbqivjln7d0c6z9f4q9"; + sha256 = "0616j0fhy6y71hyxg3n86f6hng0fmsc269s3wp4gl8ww4p8hd8f2"; libName = "portable_atomic_util"; dependencies = [ { @@ -6982,6 +7034,7 @@ rec { } ]; features = { + "serde" = [ "dep:serde" ]; "std" = [ "alloc" ]; }; resolvedDefaultFeatures = [ "alloc" ]; @@ -7265,11 +7318,11 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand 0.8.5" = rec { + "rand 0.8.6" = rec { crateName = "rand"; - version = "0.8.5"; + version = "0.8.6"; edition = "2018"; - sha256 = "013l6931nn7gkc23jz5mm3qdhf93jjf0fg64nz2lp4i51qd8vbrl"; + sha256 = "12kd4rljn86m00rcaz4c1rcya4mb4gk5ig6i8xq00a8wjgxfr82w"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -7291,12 +7344,9 @@ rec { "default" = [ "std" "std_rng" ]; "getrandom" = [ "rand_core/getrandom" ]; "libc" = [ "dep:libc" ]; - "log" = [ "dep:log" ]; - "packed_simd" = [ "dep:packed_simd" ]; "rand_chacha" = [ "dep:rand_chacha" ]; "serde" = [ "dep:serde" ]; "serde1" = [ "serde" "rand_core/serde1" ]; - "simd_support" = [ "packed_simd" ]; "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; "std_rng" = [ "rand_chacha" ]; }; @@ -8322,9 +8372,9 @@ rec { }; "rustls-webpki" = rec { crateName = "rustls-webpki"; - version = "0.103.12"; + version = "0.103.13"; edition = "2021"; - sha256 = "01nxzkfd1l96jzp04svc7iznlkarzx3wb9p63a0i17rc4y2vnyc2"; + sha256 = "0vkm7z9pnxz5qz66p2kmyy2pwx0g4jnsbqk5xzfhs4czcjl2ki31"; libName = "webpki"; dependencies = [ { @@ -9944,7 +9994,7 @@ rec { { name = "stackable-operator"; packageId = "stackable-operator"; - features = [ "crds" "webhook" ]; + features = [ "webhook" ]; } { name = "strum"; @@ -10421,6 +10471,16 @@ rec { }; resolvedDefaultFeatures = [ "i128" ]; }; + "symlink" = rec { + crateName = "symlink"; + version = "0.1.0"; + edition = "2015"; + sha256 = "02h1i0b81mxb4vns4xrvrfibpcvs7jqqav8p3yilwik8cv73r5x7"; + authors = [ + "Chris Morgan " + ]; + + }; "syn 1.0.109" = rec { crateName = "syn"; version = "1.0.109"; @@ -10853,9 +10913,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.0"; + version = "1.52.1"; edition = "2021"; - sha256 = "0xnpygq9578c8rqjgkj5bj8pgfx9zj337kvk3v4kigqwkgska4d9"; + sha256 = "1imw1dkkv38p66i33m5hsyk3d6prsbyrayjvqhndjvz89ybywzdn"; authors = [ "Tokio Contributors " ]; @@ -11694,9 +11754,9 @@ rec { }; "tracing-appender" = rec { crateName = "tracing-appender"; - version = "0.2.4"; + version = "0.2.5"; edition = "2018"; - sha256 = "1bxf7xvsr89glbq174cx0b9pinaacbhlmc85y1ssniv2rq5lhvbq"; + sha256 = "0g4a6q5s3wafid5lqw1ljzvh1nhk3a4zmb627fxv96dr7qcqc1h5"; libName = "tracing_appender"; authors = [ "Zeki Sherif " @@ -11707,6 +11767,10 @@ rec { name = "crossbeam-channel"; packageId = "crossbeam-channel"; } + { + name = "symlink"; + packageId = "symlink"; + } { name = "thiserror"; packageId = "thiserror 2.0.18"; @@ -12061,9 +12125,9 @@ rec { }; "typenum" = rec { crateName = "typenum"; - version = "1.19.0"; + version = "1.20.0"; edition = "2018"; - sha256 = "1fw2mpbn2vmqan56j1b3fbpcdg80mz26fm53fs16bq5xcq84hban"; + sha256 = "1pj35y6q11d3y55gdl6g1h2dfhmybjming0jdi9bh0bpnqm11kj0"; authors = [ "Paho Lurie-Gregg " "Andre Bogus " @@ -12295,9 +12359,9 @@ rec { }; "wasip2" = rec { crateName = "wasip2"; - version = "1.0.2+wasi-0.2.9"; + version = "1.0.3+wasi-0.2.9"; edition = "2021"; - sha256 = "1xdw7v08jpfjdg94sp4lbdgzwa587m5ifpz6fpdnkh02kwizj5wm"; + sha256 = "1mi3w855dz99xzjqc4aa8c9q5b6z1y5c963pkk4cvmr6vdr4c1i0"; dependencies = [ { name = "wit-bindgen"; @@ -13772,9 +13836,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.1"; + version = "1.0.2"; edition = "2021"; - sha256 = "1dbji1bwviy08pl74f2qw2m4w9hc4p3vyl3lfj05jdydy59w1nh9"; + sha256 = "1l7xnfvlgy4da6gq5ip2bgcm8i9d0rwzaxg1p88nlw8lxy5p1q9f"; dependencies = [ { name = "memchr"; @@ -13797,19 +13861,20 @@ rec { }; "wit-bindgen" = rec { crateName = "wit-bindgen"; - version = "0.51.0"; + version = "0.57.1"; edition = "2024"; - sha256 = "19fazgch8sq5cvjv3ynhhfh5d5x08jq2pkw8jfb05vbcyqcr496p"; + sha256 = "0vjk2jb593ri9k1aq4iqs2si9mrw5q46wxnn78im7hm7hx799gqy"; libName = "wit_bindgen"; authors = [ "Alex Crichton " ]; features = { - "async" = [ "std" "wit-bindgen-rust-macro?/async" ]; - "async-spawn" = [ "async" "dep:futures" ]; + "async-spawn" = [ "async" "dep:futures" "std" ]; "bitflags" = [ "dep:bitflags" ]; - "default" = [ "macros" "realloc" "async" "std" "bitflags" ]; + "default" = [ "macros" "realloc" "async" "std" "bitflags" "macro-string" ]; + "futures-stream" = [ "async" "dep:futures" ]; "inter-task-wakeup" = [ "async" ]; + "macro-string" = [ "wit-bindgen-rust-macro?/macro-string" ]; "macros" = [ "dep:wit-bindgen-rust-macro" ]; "rustc-dep-of-std" = [ "dep:core" "dep:alloc" ]; }; diff --git a/Cargo.toml b/Cargo.toml index 22382be4..90c61868 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/superset-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.110.1", features = ["crds", "webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.110.1", features = ["webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 2f7afef9..5df3c2ba 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -45,7 +45,7 @@ mod tests { image: productVersion: 4.1.4 clusterConfig: - credentialsSecretName: superset-db-credentials + credentialsSecretName: superset-admin-credentials metadataDatabase: postgresql: host: superset-postgresql diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index f9534a8c..0c8a0e68 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -425,7 +425,7 @@ pub async fn reconcile_superset( client, ) .await - .context(InvalidSecretKeySnafu)?; + .context(CreateSecretKeySecretSnafu)?; let mut ss_cond_builder = StatefulSetConditionBuilder::default(); From 1dfaa8e7986dc7e4087ec8654d63549b7740e1c1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 22 Apr 2026 12:39:18 +0200 Subject: [PATCH 08/11] extend changelog, use helper function --- CHANGELOG.md | 1 + rust/operator-binary/src/crd/affinity.rs | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc043509..1f4de14e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Bump `stackable-operator` to 0.110.0 and `kube` to 3.1.0 ([#719]). - Support setting `clientAuthenticationMethod` for OIDC authentication. The value is passed through to the Flask-AppBuilder config as `token_endpoint_auth_method` ([#719]). - BREAKING: Rename `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` in `superset_config.py` for arbitrary Python code to `FILE_HEADER` and `FILE_FOOTER` ([#719], [#721]). +- Use an internal secret for the secret-key ([#722]). - BREAKING: The `.clusterConfig.credentialsSecret` field has been renamed to `.clusterConfig.credentialsSecretName` for consistency ([#722]). - BREAKING: Implement generic database connection. This means you need to replace your simple database connection string with a typed struct. diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 5df3c2ba..202069d4 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -29,6 +29,7 @@ mod tests { api::core::v1::{PodAffinityTerm, PodAntiAffinity, WeightedPodAffinityTerm}, apimachinery::pkg::apis::meta::v1::LabelSelector, }, + utils::yaml_from_str_singleton_map, }; use super::*; @@ -56,10 +57,8 @@ mod tests { default: replicas: 1 "#; - let deserializer = serde_yaml::Deserializer::from_str(input); let superset: v1alpha1::SupersetCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer) - .expect("illegal test input"); + yaml_from_str_singleton_map(input).expect("illegal test input"); let merged_config = superset .merged_config(&SupersetRole::Node, &superset.node_rolegroup_ref("default")) .unwrap(); From 51c75c4f4e9b8a7229feaad3027cff564c741558 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 22 Apr 2026 12:45:01 +0200 Subject: [PATCH 09/11] remove credentials secret from product config --- deploy/config-spec/properties.yaml | 12 ------------ .../helm/superset-operator/configs/properties.yaml | 12 ------------ rust/operator-binary/src/crd/mod.rs | 5 ----- 3 files changed, 29 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index de0eee44..e063e334 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -3,18 +3,6 @@ version: 0.1.0 spec: units: [] properties: - - property: &credentialsSecretName - propertyNames: - - name: "credentialsSecretName" - kind: - type: "env" - datatype: - type: "string" - roles: - - name: "node" - required: true - asOfVersion: "0.0.0" - description: "The name of the Secret where the Superset credentials are stored." - property: &rowLimit propertyNames: - name: "ROW_LIMIT" diff --git a/deploy/helm/superset-operator/configs/properties.yaml b/deploy/helm/superset-operator/configs/properties.yaml index de0eee44..e063e334 100644 --- a/deploy/helm/superset-operator/configs/properties.yaml +++ b/deploy/helm/superset-operator/configs/properties.yaml @@ -3,18 +3,6 @@ version: 0.1.0 spec: units: [] properties: - - property: &credentialsSecretName - propertyNames: - - name: "credentialsSecretName" - kind: - type: "env" - datatype: - type: "string" - roles: - - name: "node" - required: true - asOfVersion: "0.0.0" - description: "The name of the Secret where the Superset credentials are stored." - property: &rowLimit propertyNames: - name: "ROW_LIMIT" diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 2a86f412..fddb7f56 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -444,7 +444,6 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { } impl v1alpha1::SupersetConfig { - pub const CREDENTIALS_SECRET_NAME_PROPERTY: &'static str = "credentialsSecretName"; pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; fn default_config(cluster_name: &str, role: &SupersetRole) -> v1alpha1::SupersetConfigFragment { @@ -478,10 +477,6 @@ impl Configuration for v1alpha1::SupersetConfigFragment { _role_name: &str, ) -> Result>, product_config_utils::Error> { let mut result = BTreeMap::new(); - result.insert( - v1alpha1::SupersetConfig::CREDENTIALS_SECRET_NAME_PROPERTY.to_string(), - Some(cluster.spec.cluster_config.credentials_secret_name.clone()), - ); if let Some(msec) = &cluster.spec.cluster_config.mapbox_secret { result.insert( v1alpha1::SupersetConfig::MAPBOX_SECRET_PROPERTY.to_string(), From 4fc96c9f01a3817f67ae580bec947de89f4010b4 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 22 Apr 2026 14:21:33 +0200 Subject: [PATCH 10/11] replace eval with os.path.expandvars --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- rust/operator-binary/src/config.rs | 2 +- .../src/druid_connection_controller.rs | 17 ++--------------- .../operator-binary/src/superset_controller.rs | 8 -------- 5 files changed, 21 insertions(+), 42 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 1a40390e..13e708ff 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4868,7 +4868,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "k8s_version"; authors = [ @@ -9542,7 +9542,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_certs"; authors = [ @@ -9645,7 +9645,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_operator"; authors = [ @@ -9825,7 +9825,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9860,7 +9860,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_shared"; authors = [ @@ -10047,7 +10047,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_telemetry"; authors = [ @@ -10157,7 +10157,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_versioned"; authors = [ @@ -10201,7 +10201,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10269,7 +10269,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 92bf1f24..e19b553d 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#k8s-version@0.1.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-certs@0.4.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator-derive@0.3.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator@0.110.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-shared@0.1.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-telemetry@0.6.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned-macros@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-webhook@0.9.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#k8s-version@0.1.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-certs@0.4.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator-derive@0.3.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator@0.110.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-shared@0.1.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-telemetry@0.6.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned-macros@0.9.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned@0.9.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-webhook@0.9.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 323c67af..b94dc44a 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -47,7 +47,7 @@ pub fn add_superset_config( ); config.insert( SupersetConfigOptions::SqlalchemyDatabaseUri.to_string(), - "os.environ.get('SQLALCHEMY_DATABASE_URI')".to_owned(), + "os.path.expandvars(os.environ.get('SQLALCHEMY_DATABASE_URI'))".to_owned(), ); config.insert( SupersetConfigOptions::StatsLogger.to_string(), diff --git a/rust/operator-binary/src/druid_connection_controller.rs b/rust/operator-binary/src/druid_connection_controller.rs index 944830e5..30e18a47 100644 --- a/rust/operator-binary/src/druid_connection_controller.rs +++ b/rust/operator-binary/src/druid_connection_controller.rs @@ -318,7 +318,7 @@ async fn build_import_job( ) -> Result { let mut commands = vec![]; - let config = "import os; SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')"; + let config = "import os; SQLALCHEMY_DATABASE_URI = os.path.expandvars(os.environ.get('SQLALCHEMY_DATABASE_URI'))"; commands.push(format!("mkdir -p {PYTHONPATH}")); commands.push(format!( "echo \"{config}\" > {PYTHONPATH}/{SUPERSET_CONFIG_FILENAME}" @@ -345,20 +345,7 @@ async fn build_import_job( container_builder .image_from_product_image(resolved_product_image) .command(vec!["/bin/bash".to_string()]) - .args(vec![ - String::from("-c"), - // SQLALCHEMY_DATABASE_URI contains a template with bash variable references - // (e.g. ${METADATA_DATABASE_USERNAME}) that must be resolved before Python - // reads it via os.environ. Airflow's config system calls expandvars() - // on env var values automatically (see [1]), but Superset does not - so we - // resolve them here with eval. - // [1] https://github.com/apache/airflow/blob/2.10.5/airflow/configuration.py#L1084-L1086 - // (same behaviour in 3.x: shared/configuration/src/airflow_shared/configuration/parser.py) - format!( - "export SQLALCHEMY_DATABASE_URI=$(eval echo \"$SQLALCHEMY_DATABASE_URI\")\n{}", - commands.join("; ") - ), - ]) + .args(vec![String::from("-c"), commands.join("; ")]) .add_env_var( "SQLALCHEMY_DATABASE_URI", metadata_database_connection_details.url_template.clone(), diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 0c8a0e68..ab6dc6c1 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -803,14 +803,6 @@ fn build_server_rolegroup_statefulset( {auth_commands} - # SQLALCHEMY_DATABASE_URI contains a template with bash variable references - # (e.g. ${{METADATA_DATABASE_USERNAME}}) that must be resolved before Python - # reads it via os.environ.get(). Airflow's config system calls expandvars() - # on env var values automatically (see [1]), but Superset does not - so we - # resolve them here with eval. - # [1] https://github.com/apache/airflow/blob/2.10.5/airflow/configuration.py#L1084-L1086 - # (same behaviour in 3.x: shared/configuration/src/airflow_shared/configuration/parser.py) - export SQLALCHEMY_DATABASE_URI=$(eval echo \"$SQLALCHEMY_DATABASE_URI\") superset db upgrade set +x echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' From 1acf0efa09333c869179b053731b784d0319a0d6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 22 Apr 2026 14:52:10 +0200 Subject: [PATCH 11/11] Update CHANGELOG.md Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f4de14e..25087ac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ - Bump `stackable-operator` to 0.110.0 and `kube` to 3.1.0 ([#719]). - Support setting `clientAuthenticationMethod` for OIDC authentication. The value is passed through to the Flask-AppBuilder config as `token_endpoint_auth_method` ([#719]). - BREAKING: Rename `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` in `superset_config.py` for arbitrary Python code to `FILE_HEADER` and `FILE_FOOTER` ([#719], [#721]). -- Use an internal secret for the secret-key ([#722]). +- Use an internal Secret for the Superset `SECRET_KEY`. + Going forward, the operator will automatically create the Secret in case it doesn't exist ([#722]). - BREAKING: The `.clusterConfig.credentialsSecret` field has been renamed to `.clusterConfig.credentialsSecretName` for consistency ([#722]). - BREAKING: Implement generic database connection. This means you need to replace your simple database connection string with a typed struct.