feat: Add generic database connection mechanism

crates/stackable-operator/crds/DummyCluster.yaml

@@ -72,6 +72,208 @@ spec:
                       and `stopped` will take no effect until `reconciliationPaused` is set to false or removed.
                     type: boolean
                 type: object
+              databaseConnection:
+                oneOf:
+                - required:
+                  - postgresql
+                - required:
+                  - mysql
+                - required:
+                  - derby
+                - required:
+                  - redis
+                - required:
+                  - genericJDBC
+                - required:
+                  - genericSQLAlchemy
+                - required:
+                  - genericCelery
+                properties:
+                  derby:
+                    description: |-
+                      Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database.
+
+                      Derby is an embedded, file-based Java database engine that requires no separate server process.
+                      It is typically used for development, testing, or as a lightweight metastore backend (e.g. for
+                      Apache Hive).
+                    properties:
+                      location:
+                        description: |-
+                          Path on the filesystem where Derby stores its database files.
+
+                          If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`.
+                          The `{unique_database_name}` part is automatically handled by the operator and is added to
+                          prevent clashing database files. The `create=true` flag is always appended to the JDBC URL,
+                          so the database is created automatically if it does not yet exist at this location.
+                        nullable: true
+                        type: string
+                    type: object
+                  genericCelery:
+                    description: |-
+                      A generic Celery database connection for broker or result backend types not covered by a
+                      dedicated variant.
+
+                      Use this when you need a Celery-compatible connection that does not have a first-class
+                      connection type. The complete connection URI is read from a Secret, giving the user full
+                      control over the connection string.
+                    properties:
+                      uriSecret:
+                        description: The name of the Secret that contains an `uri` key with the complete SQLAlchemy URI.
+                        type: string
+                    required:
+                    - uriSecret
+                    type: object
+                  genericJDBC:
+                    description: |-
+                      A generic JDBC database connection for database types not covered by a dedicated variant.
+
+                      Use this when you need to connect to a JDBC-compatible database that does not have a
+                      first-class connection type. You are responsible for providing the correct driver class name
+                      and a fully-formed JDBC URI as well as providing the needed classes on the Java classpath.
+                    properties:
+                      credentialsSecret:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the database.
+                        type: string
+                      driver:
+                        description: |-
+                          Fully-qualified Java class name of the JDBC driver, e.g. `org.postgresql.Driver` or
+                          `com.mysql.jdbc.Driver`. The driver JAR must be provided by you on the classpath.
+                        type: string
+                      uri:
+                        description: |-
+                          The JDBC connection URI, e.g. `jdbc:postgresql://my-host:5432/mydb`. Credentials must
+                          not be embedded in this URI; they are instead injected via environment variables sourced
+                          from `credentials_secret`.
+                        format: uri
+                        type: string
+                    required:
+                    - credentialsSecret
+                    - driver
+                    - uri
+                    type: object
+                  genericSQLAlchemy:
+                    description: |-
+                      A generic SQLAlchemy database connection for database types not covered by a dedicated variant.
+
+                      Use this when you need to connect to a SQLAlchemy-compatible database that does not have a
+                      first-class connection type. The complete connection URI is read from a Secret, giving the user
+                      full control over the connection string including any driver-specific options.
+                    properties:
+                      uriSecret:
+                        description: The name of the Secret that contains an `uri` key with the complete SQLAlchemy URI.
+                        type: string
+                    required:
+                    - uriSecret
+                    type: object
+                  mysql:
+                    description: Connection settings for a [MySQL](https://www.mysql.com/) database.
+                    properties:
+                      credentialsSecret:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the MySQL server.
+                        type: string
+                      database:
+                        description: Name of the database (schema) to connect to.
+                        type: string
+                      host:
+                        description: Hostname or IP address of the MySQL server.
+                        type: string
+                      parameters:
+                        additionalProperties:
+                          type: string
+                        default: {}
+                        description: |-
+                          Additional map of JDBC connection parameters to append to the connection URL. The given
+                          `HashMap<String, String>` will be converted to query parameters in the form of
+                          `?param1=value1&param2=value2`.
+                        type: object
+                      port:
+                        default: 3306
+                        description: Port the MySQL server is listening on. Defaults to `3306`.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                    required:
+                    - credentialsSecret
+                    - database
+                    - host
+                    type: object
+                  postgresql:
+                    description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database.
+                    properties:
+                      credentialsSecret:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the PostgreSQL server.
+                        type: string
+                      database:
+                        description: Name of the database (schema) to connect to.
+                        type: string
+                      host:
+                        description: Hostname or IP address of the PostgreSQL server.
+                        type: string
+                      parameters:
+                        additionalProperties:
+                          type: string
+                        default: {}
+                        description: |-
+                          Additional map of JDBC connection parameters to append to the connection URL. The given
+                          `HashMap<String, String>` will be converted to query parameters in the form of
+                          `?param1=value1&param2=value2`.
+                        type: object
+                      port:
+                        default: 5432
+                        description: Port the PostgreSQL server is listening on. Defaults to `5432`.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                    required:
+                    - credentialsSecret
+                    - database
+                    - host
+                    type: object
+                  redis:
+                    description: |-
+                      Connection settings for a [Redis](https://redis.io/) instance.
+
+                      Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow).
+                    properties:
+                      credentialsSecret:
+                        description: |-
+                          Name of a Secret containing the `username` and `password` keys used to authenticate
+                          against the Redis server.
+                        type: string
+                      databaseId:
+                        default: 0
+                        description: |-
+                          Numeric index of the Redis logical database to use. Defaults to `0`.
+
+                          Redis supports multiple logical databases within a single instance, identified by an
+                          integer index. Database `0` is the default.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                      host:
+                        description: Hostname or IP address of the Redis server.
+                        type: string
+                      port:
+                        default: 6379
+                        description: Port the Redis server is listening on. Defaults to `6379`.
+                        format: uint16
+                        maximum: 65535.0
+                        minimum: 0.0
+                        type: integer
+                    required:
+                    - credentialsSecret
+                    - host
+                    type: object
+                type: object
               domainName:
                 description: A validated domain name type conforming to RFC 1123, so e.g. not an IP address
                 type: string
@@ -2128,6 +2330,7 @@ spec:
             required:
             - clientAuthenticationDetails
             - clusterOperation
+            - databaseConnection
             - domainName
             - gitSync
             - hostName

crates/stackable-operator/src/builder/pod/container.rs

@@ -1,4 +1,4 @@
-use std::fmt;
+use std::{borrow::Borrow, fmt};
 
 use indexmap::IndexMap;
 use k8s_openapi::api::core::v1::{
@@ -175,8 +175,14 @@ impl ContainerBuilder {
         self
     }
 
-    pub fn add_env_vars(&mut self, env_vars: Vec<EnvVar>) -> &mut Self {
-        self.env.get_or_insert_with(Vec::new).extend(env_vars);
+    pub fn add_env_vars<I>(&mut self, env_vars: I) -> &mut Self
+    where
+        I: IntoIterator,
+        I::Item: std::borrow::Borrow<EnvVar>,
+    {
+        self.env
+            .get_or_insert_with(Vec::new)
+            .extend(env_vars.into_iter().map(|e| e.borrow().clone()));
         self
     }
 

crates/stackable-operator/src/builder/pod/env.rs

@@ -0,0 +1,20 @@
+use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector};
+
+pub fn env_var_from_secret(
+    env_var_name: impl Into<String>,
+    secret_name: impl Into<String>,
+    secret_key: impl Into<String>,
+) -> EnvVar {
+    EnvVar {
+        name: env_var_name.into(),
+        value_from: Some(EnvVarSource {
+            secret_key_ref: Some(SecretKeySelector {
+                name: secret_name.into(),
+                key: secret_key.into(),
+                ..Default::default()
+            }),
+            ..Default::default()
+        }),
+        ..Default::default()
+    }
+}

crates/stackable-operator/src/builder/pod/mod.rs

@@ -29,6 +29,7 @@ use crate::{
 };
 
 pub mod container;
+pub mod env;
 pub mod probe;
 pub mod resources;
 pub mod security;

crates/stackable-operator/src/client.rs

@@ -590,7 +590,7 @@ impl Client {
 pub trait GetApi: Resource + Sized {
     /// The namespace type for `Self`'s scope.
     ///
-    /// This will be [`str`] for namespaced resource, and [`()`] for cluster-scoped resources.
+    /// This will be [`str`] for namespaced resource, and `()` for cluster-scoped resources.
     type Namespace: ?Sized;
     /// Get a [`kube::Api`] for `Self`'s native scope..
     fn get_api(client: kube::Client, ns: &Self::Namespace) -> kube::Api<Self>

crates/stackable-operator/src/crd/git_sync/v1alpha1_impl.rs

@@ -197,7 +197,7 @@ impl GitSyncResources {
                 one_time,
                 container_log_config,
             )])
-            .add_env_vars(env_vars.into())
+            .add_env_vars(env_vars)
             .add_volume_mounts(volume_mounts.to_vec())
             .context(AddVolumeMountSnafu)?
             .resources(

crates/stackable-operator/src/crd/git_sync/v1alpha2_impl.rs

@@ -241,7 +241,7 @@ impl GitSyncResources {
                 one_time,
                 container_log_config,
             )])
-            .add_env_vars(env_vars.into())
+            .add_env_vars(env_vars)
             .add_volume_mounts(volume_mounts.to_vec())
             .context(AddVolumeMountSnafu)?
             .resources(

crates/stackable-operator/src/databases/databases/derby.rs

@@ -0,0 +1,56 @@
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use snafu::{ResultExt, Snafu};
+
+use crate::databases::{
+    TemplatingMechanism,
+    drivers::jdbc::{JDBCDatabaseConnection, JDBCDatabaseConnectionDetails},
+};
+
+#[derive(Debug, Snafu)]
+pub enum Error {
+    #[snafu(display("failed to parse connection URL"))]
+    ParseConnectionUrl { source: url::ParseError },
+}
+
+/// Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database.
+///
+/// Derby is an embedded, file-based Java database engine that requires no separate server process.
+/// It is typically used for development, testing, or as a lightweight metastore backend (e.g. for
+/// Apache Hive).
+#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DerbyConnection {
+    /// Path on the filesystem where Derby stores its database files.
+    ///
+    /// If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`.
+    /// The `{unique_database_name}` part is automatically handled by the operator and is added to
+    /// prevent clashing database files. The `create=true` flag is always appended to the JDBC URL,
+    /// so the database is created automatically if it does not yet exist at this location.
+    pub location: Option<String>,
+}
+
+impl JDBCDatabaseConnection for DerbyConnection {
+    fn jdbc_connection_details_with_templating(
+        &self,
+        unique_database_name: &str,
+        _templating_mechanism: &TemplatingMechanism,
+    ) -> Result<JDBCDatabaseConnectionDetails, crate::databases::Error> {
+        let location = self
+            .location
+            .clone()
+            .unwrap_or_else(|| format!("/tmp/derby/{unique_database_name}/derby.db"));
+        let connection_uri = format!("jdbc:derby:{location};create=true",);
+        let connection_uri = connection_uri.parse().context(ParseConnectionUrlSnafu)?;
+
+        Ok(JDBCDatabaseConnectionDetails {
+            // Sadly the Derby driver class name is a bit complicated, e.g. for HMS up to 4.1.x we used
+            // "org.apache.derby.jdbc.EmbeddedDriver",
+            // for HMS 4.2.x we used "org.apache.derby.iapi.jdbc.AutoloadedDriver".
+            driver: "org.apache.derby.jdbc.EmbeddedDriver".to_owned(),
+            connection_uri,
+            username_env: None,
+            password_env: None,
+        })
+    }
+}

crates/stackable-operator/src/databases/databases/mod.rs

@@ -0,0 +1,4 @@
+pub mod derby;
+pub mod mysql;
+pub mod postgresql;
+pub mod redis;