fix: Airflow demo improvements

demos/airflow-scheduled-job/01-airflow-demo-role.yaml

@@ -0,0 +1,45 @@
+---
+# Used by the Airflow cluster itself: the sparkapp_dag DAG creates a SparkApplication
+# (SparkKubernetesOperator) and polls its status (SparkKubernetesSensor). Reading the Spark
+# driver/executor pods and their logs is already covered by the RoleBinding the Airflow operator
+# creates for airflow-serviceaccount.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: airflow-demo-spark
+rules:
+- apiGroups:
+    - spark.stackable.tech
+  resources:
+    - sparkapplications
+  verbs:
+    - create
+    - get
+---
+# Used by the setup Jobs (start-*-job): "kubectl rollout status" on the Airflow and Kafka
+# StatefulSets and "kubectl exec" into kafka-broker-default-0 and airflow-webserver-default-0.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: airflow-demo-setup
+rules:
+- apiGroups:
+    - apps
+  resources:
+    - statefulsets
+  verbs:
+    - get
+    - list
+    - watch
+- apiGroups:
+    - ""
+  resources:
+    - pods
+  verbs:
+    - get
+- apiGroups:
+    - ""
+  resources:
+    - pods/exec
+  verbs:
+    - create

demos/airflow-scheduled-job/02-airflow-demo-rolebinding.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: airflow-demo-spark
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: airflow-demo-spark
+subjects:
+- kind: ServiceAccount
+  name: airflow-serviceaccount
+  namespace: {{ NAMESPACE }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: airflow-demo-setup
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: airflow-demo-setup
+subjects:
+- kind: ServiceAccount
+  name: demo-serviceaccount
+  namespace: {{ NAMESPACE }}

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -6,6 +6,7 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
       containers:
         - name: start-pyspark-job
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -6,6 +6,7 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
       containers:
         - name: start-date-job
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev

demos/airflow-scheduled-job/05-enable-and-run-kafka-dag.yaml

@@ -6,6 +6,7 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
       containers:
         - name: start-kafka-job
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev

demos/airflow-scheduled-job/06-create-opa-users.yaml

@@ -6,6 +6,7 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
       containers:
         - name: start-users-job
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev

demos/airflow-scheduled-job/serviceaccount.yaml

@@ -1,48 +1,7 @@
 ---
+# Runs the setup Jobs. Its permissions are granted by the airflow-demo-setup Role
+# (01-airflow-demo-role.yaml).
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: demo-serviceaccount
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: demo-clusterrolebinding
-subjects:
-  - kind: ServiceAccount
-    name: demo-serviceaccount
-    namespace: {{ NAMESPACE }}
-roleRef:
-  kind: ClusterRole
-  name: demo-clusterrole
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: demo-clusterrole
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - apps
-    resources:
-      - statefulsets
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - batch
-    resources:
-      - jobs
-    verbs:
-      - get
-      - list
-      - watch

demos/demos-v2.yaml

@@ -46,13 +46,13 @@ demos:
       - airflow
       - job-scheduling
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/01-airflow-demo-clusterrole.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/02-airflow-demo-clusterrolebinding.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/serviceaccount.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/01-airflow-demo-role.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/02-airflow-demo-rolebinding.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/05-enable-and-run-kafka-dag.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/06-create-opa-users.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/serviceaccount.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/create-trino-tables.yaml
     supportedNamespaces: []
     resourceRequests: