Skip to content

Commit 1098ab8

Browse files
adwk67adwk67
committed
reduce size of demo and prepare for public use
1 parent d4f9bca commit 1098ab8

File tree

18 files changed

+438
-1348
lines changed

18 files changed

+438
-1348
lines changed

‎stacks/airflow-trino-dbt/airflow.yaml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
---
2+
apiVersion: airflow.stackable.tech/v1alpha1
3+
kind: AirflowCluster
4+
metadata:
5+
name: airflow
6+
spec:
7+
image:
8+
productVersion: 3.0.6
9+
pullPolicy: IfNotPresent
10+
clusterConfig:
11+
vectorAggregatorConfigMapName: vector-aggregator-discovery
12+
authorization:
13+
opa:
14+
configMapName: opa
15+
package: airflow
16+
cache:
17+
entryTimeToLive: 5s
18+
maxEntries: 10
19+
loadExamples: false
20+
credentialsSecret: airflow-credentials
21+
volumes:
22+
- name: airflow-dags
23+
configMap:
24+
name: airflow-dags
25+
volumeMounts:
26+
- name: airflow-dags
27+
mountPath: /stackable/airflow/dags/dbt.py
28+
subPath: dbt.py
29+
webservers:
30+
roleConfig:
31+
listenerClass: external-stable
32+
config:
33+
logging:
34+
enableVectorAgent: true
35+
resources:
36+
cpu:
37+
min: "2"
38+
max: "3"
39+
memory:
40+
limit: 3Gi
41+
roleGroups:
42+
default:
43+
replicas: 1
44+
celeryExecutors:
45+
config:
46+
logging:
47+
enableVectorAgent: true
48+
resources:
49+
cpu:
50+
min: "2"
51+
max: "3"
52+
memory:
53+
limit: 4Gi
54+
roleGroups:
55+
default:
56+
replicas: 1
57+
# kubernetesExecutors:
58+
# config: {}
59+
schedulers:
60+
config:
61+
logging:
62+
enableVectorAgent: true
63+
roleGroups:
64+
default:
65+
replicas: 1
66+
dagProcessors:
67+
config:
68+
logging:
69+
enableVectorAgent: true
70+
roleGroups:
71+
default:
72+
replicas: 1
73+
triggerers:
74+
config:
75+
logging:
76+
enableVectorAgent: true
77+
roleGroups:
78+
default:
79+
replicas: 1
80+
---
81+
apiVersion: v1
82+
kind: ConfigMap
83+
metadata:
84+
name: airflow-dags
85+
data:
86+
dbt.py: |
87+
from airflow import DAG
88+
from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
89+
from kubernetes.client import models as k8s
90+
from kubernetes.client import V1EnvVar, V1EnvVarSource, V1SecretKeySelector
91+
92+
tls_volume = k8s.V1Volume(
93+
name="server-tls-mount",
94+
ephemeral=k8s.V1EphemeralVolumeSource(
95+
volume_claim_template=k8s.V1PersistentVolumeClaimTemplate(
96+
metadata=k8s.V1ObjectMeta(
97+
annotations={
98+
"secrets.stackable.tech/class": "trino-tls",
99+
"secrets.stackable.tech/scope": "pod,node"
100+
}
101+
),
102+
spec=k8s.V1PersistentVolumeClaimSpec(
103+
access_modes=["ReadWriteOnce"],
104+
resources=k8s.V1ResourceRequirements(
105+
requests={"storage": "1"}
106+
),
107+
storage_class_name="secrets.stackable.tech"
108+
)
109+
)
110+
)
111+
)
112+
113+
tls_volume_mount = k8s.V1VolumeMount(
114+
name="server-tls-mount", mount_path="/dbt/trusted"
115+
)
116+
117+
pod_security_context = k8s.V1PodSecurityContext(
118+
fs_group=1000
119+
)
120+
121+
with DAG(
122+
dag_id="run_dbt",
123+
schedule=None,
124+
tags=["Demo", "DBT"],
125+
catchup=False
126+
) as dag:
127+
run_dbt = KubernetesPodOperator(
128+
image="oci.stackable.tech/sandbox/andrew/dbt-trino:0.0.1",
129+
image_pull_policy="IfNotPresent",
130+
cmds=["/bin/bash", "-x", "-euo", "pipefail", "-c"],
131+
arguments=["cd /dbt && export DBT_PROFILES_DIR=/dbt && dbt debug"],
132+
name="run-dbt",
133+
task_id="dbt-test",
134+
get_logs=True,
135+
volumes=[tls_volume],
136+
volume_mounts=[tls_volume_mount],
137+
env_vars=[
138+
V1EnvVar(
139+
name="TRINO_PASSWORD",
140+
value_from=V1EnvVarSource(
141+
secret_key_ref=V1SecretKeySelector(
142+
name="demo-password",
143+
key="password"
144+
)
145+
)
146+
),
147+
V1EnvVar(name="TRINO_USER", value="admin"),
148+
V1EnvVar(name="TRINO_HOST", value="trino-coordinator-default-headless.default.svc.cluster.local"),
149+
V1EnvVar(name="TRINO_PORT", value="8443"),
150+
V1EnvVar(name="CERT_PATH", value="/dbt/trusted/ca.crt"),
151+
],
152+
security_context=pod_security_context,
153+
startup_timeout_seconds=600
154+
)
155+
run_dbt

‎stacks/airflow-trino-dbt/hive-metastores.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
apiVersion: hive.stackable.tech/v1alpha1
3+
kind: HiveCluster
4+
metadata:
5+
name: hive-iceberg
6+
spec:
7+
image:
8+
productVersion: 4.1.0
9+
clusterConfig:
10+
database:
11+
connString: jdbc:postgresql://postgresql-hive-iceberg:5432/hive
12+
dbType: postgres
13+
credentialsSecret: postgres-credentials
14+
s3:
15+
reference: minio
16+
metastore:
17+
roleGroups:
18+
default:
19+
replicas: 1
20+
---
21+
apiVersion: v1
22+
kind: Secret
23+
metadata:
24+
name: postgres-credentials
25+
type: Opaque
26+
stringData:
27+
username: hive
28+
password: hive

stacks/trino-iceberg/opa-rules.yaml renamed to stacks/airflow-trino-dbt/opa-rules.yaml

File renamed without changes.

stacks/trino-iceberg/opa.yaml renamed to stacks/airflow-trino-dbt/opa.yaml

File renamed without changes.

stacks/trino-iceberg/rbac.yaml renamed to stacks/airflow-trino-dbt/rbac.yaml

File renamed without changes.

stacks/trino-iceberg/redis-airflow.yaml renamed to stacks/airflow-trino-dbt/redis-airflow.yaml

File renamed without changes.

‎stacks/airflow-trino-dbt/s3-connection.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
apiVersion: s3.stackable.tech/v1alpha1
3+
kind: S3Connection
4+
metadata:
5+
name: minio
6+
spec:
7+
host: minio.default.svc.cluster.local
8+
port: 9000
9+
accessStyle: Path
10+
credentials:
11+
secretClass: minio-s3-credentials
12+
tls:
13+
verification:
14+
server:
15+
caCert:
16+
secretClass: tls
17+
---
18+
apiVersion: secrets.stackable.tech/v1alpha1
19+
kind: SecretClass
20+
metadata:
21+
name: minio-s3-credentials
22+
spec:
23+
backend:
24+
k8sSearch:
25+
searchNamespace:
26+
pod: {}

‎stacks/airflow-trino-dbt/trino.yaml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
---
2+
apiVersion: trino.stackable.tech/v1alpha1
3+
kind: TrinoCluster
4+
metadata:
5+
name: trino
6+
spec:
7+
image:
8+
productVersion: "477"
9+
clusterConfig:
10+
catalogLabelSelector:
11+
matchLabels:
12+
trino: trino
13+
authentication:
14+
- authenticationClass: trino-users
15+
authorization:
16+
opa:
17+
configMapName: opa
18+
package: trino
19+
tls:
20+
serverSecretClass: trino-tls
21+
internalSecretClass: trino-internal-tls
22+
coordinators:
23+
roleGroups:
24+
default:
25+
replicas: 1
26+
roleConfig:
27+
listenerClass: external-stable
28+
workers:
29+
roleGroups:
30+
default:
31+
replicas: 1
32+
---
33+
apiVersion: secrets.stackable.tech/v1alpha1
34+
kind: SecretClass
35+
metadata:
36+
name: trino-tls
37+
spec:
38+
backend:
39+
autoTls:
40+
ca:
41+
secret:
42+
name: secret-provisioner-trino-tls-ca
43+
namespace: default
44+
autoGenerate: true
45+
---
46+
apiVersion: secrets.stackable.tech/v1alpha1
47+
kind: SecretClass
48+
metadata:
49+
name: trino-internal-tls
50+
spec:
51+
backend:
52+
autoTls:
53+
ca:
54+
secret:
55+
name: secret-provisioner-trino-internal-tls-ca
56+
namespace: default
57+
autoGenerate: true
58+
---
59+
apiVersion: authentication.stackable.tech/v1alpha1
60+
kind: AuthenticationClass
61+
metadata:
62+
name: trino-users
63+
spec:
64+
provider:
65+
static:
66+
userCredentialsSecret:
67+
name: trino-users
68+
---
69+
apiVersion: trino.stackable.tech/v1alpha1
70+
kind: TrinoCatalog
71+
metadata:
72+
name: iceberg
73+
labels:
74+
trino: trino
75+
spec:
76+
connector:
77+
iceberg:
78+
metastore:
79+
configMap: hive-iceberg
80+
s3:
81+
reference: minio
82+
---
83+
apiVersion: trino.stackable.tech/v1alpha1
84+
kind: TrinoCatalog
85+
metadata:
86+
name: dpx
87+
labels:
88+
trino: trino
89+
spec:
90+
connector:
91+
generic:
92+
connectorName: postgresql
93+
properties:
94+
connection-url:
95+
value: jdbc:postgresql://140.238.212.205:55432/suva
96+
connection-user:
97+
valueFromSecret:
98+
name: dpx-credentials
99+
key: username
100+
connection-password:
101+
valueFromSecret:
102+
name: dpx-credentials
103+
key: password
104+
---
105+
apiVersion: v1
106+
kind: ConfigMap
107+
metadata:
108+
name: trino-opa-bundle
109+
labels:
110+
opa.stackable.tech/bundle: "trino"
111+
data:
112+
trino.rego: |
113+
package trino
114+
115+
default allow = false
116+
117+
# Allow non-batched access
118+
allow if {
119+
is_admin
120+
}
121+
# Allow batched access
122+
batch contains i if {
123+
some i
124+
input.action.filterResources[i]
125+
is_admin
126+
}
127+
# Corner case: filtering columns is done with a single table item, and many columns inside
128+
batch contains i if {
129+
some i
130+
input.action.operation == "FilterColumns"
131+
count(input.action.filterResources) == 1
132+
input.action.filterResources[0].table.columns[i]
133+
is_admin
134+
}
135+
136+
is_admin() if {
137+
input.context.identity.user == "admin"
138+
}

0 commit comments

Comments
 (0)