feat: Add generic database connection mechanism

Cargo.toml

@@ -33,5 +33,5 @@ tokio = { version = "1.40", features = ["full"] }
 tracing = "0.1"
 
 [patch."https://github.com/stackabletech/operator-rs.git"]
-# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
+stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "spike/generic-databases" }
 # stackable-operator = { path = "../operator-rs/crates/stackable-operator" }

docs/modules/airflow/examples/example-airflow-dags-configmap.yaml

@@ -7,9 +7,12 @@ spec:
   image:
     productVersion: 3.1.6
   clusterConfig:
-    loadExamples: false
-    exposeConfig: false
-    credentialsSecret: simple-airflow-credentials
+    credentialsSecret: airflow-admin-credentials
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
     volumes:
       - name: cm-dag # <3>
         configMap:
@@ -27,14 +30,4 @@ spec:
           AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8>
         replicas: 1
   celeryExecutors:
-    roleGroups:
-      default:
-        envOverrides:
-          AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8>
-        replicas: 2
-  schedulers:
-    roleGroups:
-      default:
-        envOverrides:
-          AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8>
-        replicas: 1
+    # ...

docs/modules/airflow/examples/example-airflow-gitsync-https.yaml

@@ -9,15 +9,15 @@ spec:
   clusterConfig:
     loadExamples: false
     exposeConfig: false
-    credentialsSecret: test-airflow-credentials # <1>
+    credentialsSecret: airflow-admin-credentials # <1>
     dagsGitSync: # <2>
       - repo: https://github.com/stackabletech/airflow-operator # <3>
         branch: "main" # <4>
         gitFolder: "tests/templates/kuttl/mount-dags-gitsync/dags" # <5>
         depth: 10 # <6>
         wait: 20s # <7>
         credentials:
-          basicAuthSecretName: git-credentials # <8>
+          basicAuthSecretName: airflow-git-credentials # <8>
         gitSyncConf: # <9>
           --rev: HEAD # <10>
           # --rev: git-sync-tag # N.B. tag must be covered by "depth" (the number of commits to clone)
@@ -29,7 +29,7 @@ spec:
 apiVersion: v1
 kind: Secret
 metadata:
-  name: git-credentials # <8>
+  name: airflow-git-credentials # <8>
 type: Opaque
 data:
   user: c3Rh...

docs/modules/airflow/examples/example-airflow-incluster.yaml

@@ -9,28 +9,39 @@ spec:
   clusterConfig:
     loadExamples: false
     exposeConfig: false
-    credentialsSecret: simple-airflow-credentials
+    credentialsSecret: airflow-admin-credentials
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
   webservers:
     roleConfig:
       listenerClass: external-unstable
     roleGroups:
       default:
-        envOverrides:
+        envOverrides: &envOverrides
           AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
         replicas: 1
   schedulers:
     roleGroups:
       default:
-        envOverrides:
-          AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
+        envOverrides: *envOverrides
         replicas: 1
   celeryExecutors:
+    celeryResultBackend:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
+    celeryBroker:
+      redis:
+        host: airflow-redis-master
+        credentialsSecret: airflow-redis-credentials
     roleGroups:
       default:
-        envOverrides:
-          AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
+        envOverrides: *envOverrides
         replicas: 1
 # in case of using kubernetesExecutors
 #  kubernetesExecutors:
-#    envOverrides:
-#      AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
+#    envOverrides: *envOverrides

docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml

@@ -2,15 +2,27 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: simple-airflow-credentials
+  name: airflow-admin-credentials
 type: Opaque
 stringData:
   adminUser.username: airflow
   adminUser.firstname: Airflow
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
-  # Only needed when using celery workers (instead of Kubernetes executors)
-  connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
-  connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-postgresql-credentials
+stringData:
+  username: airflow
+  password: airflow
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-redis-credentials
+stringData:
+  username: ""
+  password: redis

docs/modules/airflow/examples/getting_started/code/airflow.yaml

@@ -10,14 +10,28 @@ spec:
   clusterConfig:
     loadExamples: true
     exposeConfig: false
-    credentialsSecret: simple-airflow-credentials
+    credentialsSecret: airflow-admin-credentials
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
   webservers:
     roleConfig:
       listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1
   celeryExecutors:
+    celeryResultBackend:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
+    celeryBroker:
+      redis:
+        host: airflow-redis-master
+        credentialsSecret: airflow-redis-credentials
     roleGroups:
       default:
         replicas: 1

docs/modules/airflow/pages/usage-guide/logging.adoc

@@ -23,6 +23,7 @@ spec:
               "flask_appbuilder":
                 level: WARN
   celeryExecutors:
+    ...
     config:
       logging:
         enableVectorAgent: true

docs/modules/airflow/pages/usage-guide/storage-resources.adoc

@@ -27,6 +27,7 @@ spec:
       default:
         replicas: 2
   celeryExecutors:
+    ...
     config:
       resources:
         cpu:

docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc

@@ -12,6 +12,7 @@ E.g. you would change the following example
 ----
 spec:
   celeryExecutors:
+    ...
     roleGroups:
       default:
         replicas: 2