From 33a9c3da349e838c24547c317bd9f7462b66cb0f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 26 Feb 2026 15:18:52 +0100 Subject: [PATCH 01/42] wip - add roles & cleanup resource structure --- extra/crds.yaml | 1536 ++++++++++++++--- rust/operator-binary/src/crd/mod.rs | 41 +- .../src/druid_connection_controller.rs | 2 +- rust/operator-binary/src/main.rs | 4 +- rust/operator-binary/src/operations/pdb.rs | 10 + .../src/resources/configmap.rs | 178 ++ .../src/resources/deployment.rs | 0 .../src/{ => resources}/listener.rs | 0 rust/operator-binary/src/resources/mod.rs | 6 + .../src/{ => resources}/rbac.rs | 0 .../src/{ => resources}/service.rs | 0 .../src/resources/statefulset.rs | 542 ++++++ .../src/superset_controller.rs | 708 +------- 13 files changed, 2124 insertions(+), 903 deletions(-) create mode 100644 rust/operator-binary/src/resources/configmap.rs create mode 100644 rust/operator-binary/src/resources/deployment.rs rename rust/operator-binary/src/{ => resources}/listener.rs (100%) create mode 100644 rust/operator-binary/src/resources/mod.rs rename rust/operator-binary/src/{ => resources}/rbac.rs (100%) rename rust/operator-binary/src/{ => resources}/service.rs (100%) create mode 100644 rust/operator-binary/src/resources/statefulset.rs diff --git a/extra/crds.yaml b/extra/crds.yaml index 09ea680d..94e49ead 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -26,224 +26,1344 @@ spec: Find more information on how to use it and the resources that the operator generates in the [operator documentation](https://docs.stackable.tech/home/nightly/superset/). properties: + beat: + description: |- + This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing + all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable + at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured + on role level. There is also a second form of config, which can only be configured + at role level, the `roleConfig`. + You can learn more about this in the + [Roles and role group concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups). + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + type: string + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object clusterConfig: description: |- - Settings that affect all roles and role groups. - The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. + Settings that affect all roles and role groups. + The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. + properties: + authentication: + default: [] + description: List of AuthenticationClasses used to authenticate users. + items: + properties: + authenticationClass: + description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users + type: string + oidc: + description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. + nullable: true + properties: + clientCredentialsSecret: + description: |- + A reference to the OIDC client credentials secret. The secret contains + the client id and secret. + type: string + extraScopes: + default: [] + description: An optional list of extra scopes which get merged with the scopes defined in the AuthenticationClass + items: + type: string + type: array + required: + - clientCredentialsSecret + type: object + syncRolesAt: + default: Registration + description: |- + If we should replace ALL the user's roles each login, or only on registration. + Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` + enum: + - Registration + - Login + type: string + userRegistration: + default: true + description: |- + Allow users who are not already in the FAB DB. + Gets mapped to `AUTH_USER_REGISTRATION` + type: boolean + userRegistrationRole: + default: Public + description: |- + This role will be given in addition to any AUTH_ROLES_MAPPING. + Gets mapped to `AUTH_USER_REGISTRATION_ROLE` + type: string + required: + - authenticationClass + type: object + type: array + authorization: + description: |- + Authorization options for Superset. + + Currently only role assignment is supported. This means that roles are assigned to users in + OPA but, due to the way Superset is implemented, the database also needs to be updated + to reflect these assignments. + Therefore, user roles and permissions must already exist in the Superset database before + they can be assigned to a user. + Warning: Any user roles assigned with the Superset UI are discarded. + nullable: true + properties: + roleMappingFromOpa: + description: |- + Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + and the name of the Rego package containing your authorization rules. + Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) + to learn how to deploy Rego authorization rules with OPA. + properties: + cache: + default: + entryTimeToLive: 30s + maxEntries: 10000 + description: Configuration for an Superset internal cache for calls to OPA + properties: + entryTimeToLive: + default: 30s + description: Time to live per entry + type: string + maxEntries: + default: 10000 + description: |- + Maximum number of entries in the cache; If this threshold is reached then the least + recently used item is removed. + format: uint32 + minimum: 0.0 + type: integer + type: object + configMapName: + description: |- + The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + for the OPA stacklet that should be used for authorization requests. + type: string + package: + description: The name of the Rego package containing the Rego rules for the product. + nullable: true + type: string + required: + - configMapName + type: object + required: + - roleMappingFromOpa + type: object + clusterOperation: + default: + reconciliationPaused: false + stopped: false + description: Cluster operations like pause reconciliation or cluster stop. + properties: + reconciliationPaused: + default: false + description: |- + Flag to stop cluster reconciliation by the operator. This means that all changes in the + custom resource spec are ignored until this flag is set to false or removed. The operator + will however still watch the deployed resources at the time and update the custom resource + status field. + If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over + `stopped` and stop the reconciliation immediately. + type: boolean + stopped: + default: false + description: |- + Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, + ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 + and therefore stopped and removed. + If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation + and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. + type: boolean + type: object + credentialsSecret: + description: |- + The name of the Secret object containing the admin user credentials and database connection details. + Read the + [getting started guide first steps](https://docs.stackable.tech/home/nightly/superset/getting_started/first_steps) + to find out more. + type: string + mapboxSecret: + description: |- + The name of a Secret object. + The Secret should contain a key `connections.mapboxApiKey`. + This is the API key required for map charts to work that use mapbox. + The token should be in the JWT format. + nullable: true + type: string + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + nullable: true + type: string + required: + - credentialsSecret + type: object + image: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Overwrite the docker image. + Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + If not specified, the operator will use its own version, e.g. `23.4.1`. + When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string + type: object + nodes: + description: |- + This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing + all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable + at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured + on role level. There is also a second form of config, which can only be configured + at role level, the `roleConfig`. + You can learn more about this in the + [Roles and role group concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups). + nullable: true properties: - authentication: - default: [] - description: List of AuthenticationClasses used to authenticate users. - items: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + type: string + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: properties: - authenticationClass: - description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users - type: string - oidc: - description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. - nullable: true + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} properties: - clientCredentialsSecret: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null description: |- - A reference to the OIDC client credentials secret. The secret contains - the client id and secret. + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true type: string - extraScopes: - default: [] - description: An optional list of extra scopes which get merged with the scopes defined in the AuthenticationClass - items: - type: string - type: array - required: - - clientCredentialsSecret + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer type: object - syncRolesAt: - default: Registration + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} description: |- - If we should replace ALL the user's roles each login, or only on registration. - Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` - enum: - - Registration - - Login - type: string - userRegistration: - default: true + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} description: |- - Allow users who are not already in the FAB DB. - Gets mapped to `AUTH_USER_REGISTRATION` - type: boolean - userRegistrationRole: - default: Public + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} description: |- - This role will be given in addition to any AUTH_ROLES_MAPPING. - Gets mapped to `AUTH_USER_REGISTRATION_ROLE` - type: string - required: - - authenticationClass + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer type: object - type: array - authorization: - description: |- - Authorization options for Superset. - - Currently only role assignment is supported. This means that roles are assigned to users in - OPA but, due to the way Superset is implemented, the database also needs to be updated - to reflect these assignments. - Therefore, user roles and permissions must already exist in the Superset database before - they can be assigned to a user. - Warning: Any user roles assigned with the Superset UI are discarded. - nullable: true - properties: - roleMappingFromOpa: - description: |- - Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) - and the name of the Rego package containing your authorization rules. - Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) - to learn how to deploy Rego authorization rules with OPA. - properties: - cache: - default: - entryTimeToLive: 30s - maxEntries: 10000 - description: Configuration for an Superset internal cache for calls to OPA - properties: - entryTimeToLive: - default: 30s - description: Time to live per entry - type: string - maxEntries: - default: 10000 - description: |- - Maximum number of entries in the cache; If this threshold is reached then the least - recently used item is removed. - format: uint32 - minimum: 0.0 - type: integer - type: object - configMapName: - description: |- - The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) - for the OPA stacklet that should be used for authorization requests. - type: string - package: - description: The name of the Rego package containing the Rego rules for the product. - nullable: true - type: string - required: - - configMapName - type: object - required: - - roleMappingFromOpa - type: object - clusterOperation: - default: - reconciliationPaused: false - stopped: false - description: Cluster operations like pause reconciliation or cluster stop. - properties: - reconciliationPaused: - default: false - description: |- - Flag to stop cluster reconciliation by the operator. This means that all changes in the - custom resource spec are ignored until this flag is set to false or removed. The operator - will however still watch the deployed resources at the time and update the custom resource - status field. - If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over - `stopped` and stop the reconciliation immediately. - type: boolean - stopped: - default: false - description: |- - Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, - ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 - and therefore stopped and removed. - If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation - and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. - type: boolean type: object - credentialsSecret: - description: |- - The name of the Secret object containing the admin user credentials and database connection details. - Read the - [getting started guide first steps](https://docs.stackable.tech/home/nightly/superset/getting_started/first_steps) - to find out more. - type: string - mapboxSecret: - description: |- - The name of a Secret object. - The Secret should contain a key `connections.mapboxApiKey`. - This is the API key required for map charts to work that use mapbox. - The token should be in the JWT format. - nullable: true - type: string - vectorAggregatorConfigMapName: - description: |- - Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). - It must contain the key `ADDRESS` with the address of the Vector aggregator. - Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) - to learn how to configure log aggregation with Vector. - nullable: true - type: string required: - - credentialsSecret + - roleGroups type: object - image: - anyOf: - - required: - - custom - - productVersion - - required: - - productVersion + objectOverrides: + default: [] description: |- - Specify which image to use, the easiest way is to only configure the `productVersion`. - You can also configure a custom image registry to pull from, as well as completely custom - images. + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. - Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) - for details. - properties: - custom: - description: |- - Overwrite the docker image. - Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` - type: string - productVersion: - description: Version of the product, e.g. `1.4.1`. - type: string - pullPolicy: - default: Always - description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' - enum: - - IfNotPresent - - Always - - Never - type: string - pullSecrets: - description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' - items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. - properties: - name: - description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - name - type: object - nullable: true - type: array - repo: - description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` - nullable: true - type: string - stackableVersion: - description: |- - Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - If not specified, the operator will use its own version, e.g. `23.4.1`. - When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. - nullable: true - type: string - type: object - nodes: + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + workers: description: |- This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable @@ -796,20 +1916,6 @@ spec: required: - roleGroups type: object - objectOverrides: - default: [] - description: |- - A list of generic Kubernetes objects, which are merged into the objects that the operator - creates. - - List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. - - Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) - for more information. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array required: - clusterConfig - image diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 113b03a1..0317420e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -35,7 +35,7 @@ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ crd::v1alpha1::{SupersetConfigFragment, SupersetRoleConfig}, - listener::default_listener_class, + resources::listener::default_listener_class, }; pub mod affinity; @@ -155,6 +155,14 @@ pub mod versioned { // no doc - docs in the struct. #[serde(default, skip_serializing_if = "Option::is_none")] pub nodes: Option>, + + // no doc - docs in the struct. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workers: Option>, + + // no doc - docs in the struct. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub beat: Option>, } // TODO: move generic version to op-rs? @@ -354,6 +362,10 @@ pub struct Connections { pub enum SupersetRole { #[strum(serialize = "node")] Node, + #[strum(serialize = "worker")] + Worker, + #[strum(serialize = "beat")] + Beat, } impl SupersetRole { @@ -364,8 +376,17 @@ impl SupersetRole { .nodes .to_owned() .map(|node| node.role_config.listener_class), + Self::Worker | Self::Beat => None, } } + + pub fn roles() -> Vec { + let mut roles = vec![]; + for role in Self::iter() { + roles.push(role.to_string()) + } + roles + } } /// A reference to a [`v1alpha1::SupersetCluster`] @@ -448,6 +469,7 @@ impl v1alpha1::SupersetConfig { pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; fn default_config(cluster_name: &str, role: &SupersetRole) -> v1alpha1::SupersetConfigFragment { + // TODO: Match for roles v1alpha1::SupersetConfigFragment { resources: ResourcesFragment { cpu: CpuLimitsFragment { @@ -539,6 +561,7 @@ impl v1alpha1::SupersetCluster { "{cluster_name}-{role}", cluster_name = self.name_any() )), + SupersetRole::Worker | SupersetRole::Beat => None, } } @@ -547,9 +570,7 @@ impl v1alpha1::SupersetCluster { } pub fn get_role_config(&self, role: &SupersetRole) -> Option<&SupersetRoleConfig> { - match role { - SupersetRole::Node => self.spec.nodes.as_ref().map(|c| &c.role_config), - } + self.get_role(role).as_ref().map(|c| &c.role_config) } pub fn get_role( @@ -558,6 +579,8 @@ impl v1alpha1::SupersetCluster { ) -> Option<&Role> { match role { SupersetRole::Node => self.spec.nodes.as_ref(), + SupersetRole::Worker => self.spec.workers.as_ref(), + SupersetRole::Beat => self.spec.beat.as_ref(), } } @@ -590,12 +613,10 @@ impl v1alpha1::SupersetCluster { // Initialize the result with all default values as baseline let conf_defaults = v1alpha1::SupersetConfig::default_config(&self.name_any(), role); - let role = match role { - SupersetRole::Node => self.spec.nodes.as_ref().context(UnknownSupersetRoleSnafu { - role: role.to_string(), - roles: vec![role.to_string()], - })?, - }; + let role = self.get_role(role).context(UnknownSupersetRoleSnafu { + role: role.to_string(), + roles: SupersetRole::roles(), + })?; // Retrieve role resource config let mut conf_role = role.config.config.to_owned(); diff --git a/rust/operator-binary/src/druid_connection_controller.rs b/rust/operator-binary/src/druid_connection_controller.rs index 26322a3f..8c6179cc 100644 --- a/rust/operator-binary/src/druid_connection_controller.rs +++ b/rust/operator-binary/src/druid_connection_controller.rs @@ -27,7 +27,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ APP_NAME, OPERATOR_NAME, crd::{PYTHONPATH, SUPERSET_CONFIG_FILENAME, druidconnection, v1alpha1}, - rbac, + resources::rbac, superset_controller::DOCKER_IMAGE_BASE_NAME, util::{JobState, get_job_state}, }; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index d5e491e5..d947bbc6 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -49,11 +49,9 @@ mod config; mod controller_commons; mod crd; mod druid_connection_controller; -mod listener; mod operations; mod product_logging; -mod rbac; -mod service; +mod resources; mod superset_controller; mod util; mod webhooks; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index e8b02fd0..15275de2 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -36,6 +36,8 @@ pub async fn add_pdbs( } let max_unavailable = pdb.max_unavailable.unwrap_or(match role { SupersetRole::Node => max_unavailable_nodes(), + SupersetRole::Worker => max_unavailable_workers(), + SupersetRole::Beat => max_unavailable_beat(), }); let pdb = PodDisruptionBudgetBuilder::new_with_role( superset, @@ -61,3 +63,11 @@ pub async fn add_pdbs( fn max_unavailable_nodes() -> u16 { 1 } + +fn max_unavailable_workers() -> u16 { + 1 +} + +fn max_unavailable_beat() -> u16 { + 1 +} diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs new file mode 100644 index 00000000..2ea4f137 --- /dev/null +++ b/rust/operator-binary/src/resources/configmap.rs @@ -0,0 +1,178 @@ +use std::{ + collections::{BTreeMap, HashMap}, + io::Write, +}; + +use product_config::{ + flask_app_config_writer::{self}, + types::PropertyNameKind, +}; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::ConfigMap, + product_config_utils::{CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY}, + product_logging::spec::Logging, + role_utils::RoleGroupRef, +}; + +use crate::{ + authorization::opa::{OPA_IMPORTS, SupersetOpaConfigResolved}, + config::{self, PYTHON_IMPORTS}, + crd::{ + SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, + authentication::SupersetClientAuthenticationDetailsResolved, + v1alpha1::{Container, SupersetCluster}, + }, + product_logging::extend_config_map_with_log_config, + superset_controller::SUPERSET_CONTROLLER_NAME, + util::build_recommended_labels, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to add Superset config settings"))] + AddSupersetConfig { source: crate::config::Error }, + + #[snafu(display( + "failed to write to String (Vec to be precise) containing superset config" + ))] + WriteToConfigFileString { source: std::io::Error }, + + #[snafu(display("failed to build Metadata"))] + BuildMetadata { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build config file for {rolegroup}"))] + BuildRoleGroupConfigFile { + source: flask_app_config_writer::FlaskAppConfigWriterError, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to build ConfigMap for {rolegroup}"))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] +pub fn build_rolegroup_config_map( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, + rolegroup_config: &HashMap>, + authentication_config: &SupersetClientAuthenticationDetailsResolved, + superset_opa_config: &Option, + logging: &Logging, +) -> Result { + let mut config_properties = BTreeMap::new(); + let mut imports = PYTHON_IMPORTS.to_vec(); + // TODO: this is true per default for versions 3.0.0 and up. + // We deactivate it here to keep existing functionality. + // However this is a security issue and should be configured properly + // Issue: https://github.com/stackabletech/superset-operator/issues/416 + config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); + + config::add_superset_config(&mut config_properties, authentication_config) + .context(AddSupersetConfigSnafu)?; + + // Adding opa configuration properties to config_properties. + // This will be injected as key/value pair in superset_config.py + if let Some(opa_config) = superset_opa_config { + // If opa role mapping is configured, insert CustomOpaSecurityManager import + imports.extend(OPA_IMPORTS); + + config_properties.extend(opa_config.as_config()); + } + + // The order here should be kept in order to preserve overrides. + // No properties should be added after this extend. + config_properties.extend( + rolegroup_config + .get(&PropertyNameKind::File( + SUPERSET_CONFIG_FILENAME.to_string(), + )) + .cloned() + .unwrap_or_default(), + ); + + let mut config_file = Vec::new(); + + // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; + } + let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + + flask_app_config_writer::write::( + &mut config_file, + config_properties.iter(), + &imports, + ) + .with_context(|_| BuildRoleGroupConfigFileSnafu { + rolegroup: rolegroup.clone(), + })?; + + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; + } + + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(BuildMetadataSnafu)? + .build(), + ) + .add_data( + SUPERSET_CONFIG_FILENAME, + String::from_utf8(config_file).unwrap(), + ); + + extend_config_map_with_log_config( + rolegroup, + logging, + &Container::Superset, + &Container::Vector, + &mut cm_builder, + ) + .context(InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + })?; + + cm_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + }) +} diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs new file mode 100644 index 00000000..e69de29b diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/resources/listener.rs similarity index 100% rename from rust/operator-binary/src/listener.rs rename to rust/operator-binary/src/resources/listener.rs diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs new file mode 100644 index 00000000..40970ec8 --- /dev/null +++ b/rust/operator-binary/src/resources/mod.rs @@ -0,0 +1,6 @@ +pub mod configmap; +pub mod deployment; +pub mod listener; +pub mod rbac; +pub mod service; +pub mod statefulset; diff --git a/rust/operator-binary/src/rbac.rs b/rust/operator-binary/src/resources/rbac.rs similarity index 100% rename from rust/operator-binary/src/rbac.rs rename to rust/operator-binary/src/resources/rbac.rs diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/resources/service.rs similarity index 100% rename from rust/operator-binary/src/service.rs rename to rust/operator-binary/src/resources/service.rs diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs new file mode 100644 index 00000000..f849027c --- /dev/null +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -0,0 +1,542 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use indoc::formatdoc; +use product_config::types::PropertyNameKind; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, + container::ContainerBuilder, + probe::ProbeBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, + }, + }, + }, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{StatefulSet, StatefulSetSpec}, + core::v1::EnvVar, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kvp::{Label, Labels}, + product_logging::{ + self, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, + }, + role_utils::RoleGroupRef, + shared::time::Duration, + utils::COMMON_BASH_TRAP_FUNCTIONS, +}; + +use crate::{ + commands::add_cert_to_python_certifi_command, + controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + crd::{ + APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, + SupersetConfigOptions, SupersetRole, + authentication::{ + SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, + }, + v1alpha1::{Container, SupersetCluster, SupersetConfig}, + }, + operations::graceful_shutdown::add_graceful_shutdown_config, + product_logging::LOG_CONFIG_FILE, + resources::listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, + superset_controller::SUPERSET_CONTROLLER_NAME, + util::build_recommended_labels, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object defines no node role"))] + NoNodeRole, + + #[snafu(display("object defines no node role-group"))] + NoNodeRoleGroup, + + #[snafu(display("invalid container name"))] + InvalidContainerName { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display( + "failed to get the {SUPERSET_CONFIG_FILENAME} file from node or product config" + ))] + MissingSupersetConfigInNodeConfig, + + #[snafu(display("failed to get {timeout} from {SUPERSET_CONFIG_FILENAME} file. It should be set in the product config or by user input", timeout = SupersetConfigOptions::SupersetWebserverTimeout))] + MissingWebServerTimeoutInSupersetConfig, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { + source: product_logging::framework::LoggingError, + }, + + #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] + AddLdapVolumesAndVolumeMounts { + source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, + }, + + #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] + AddTlsVolumesAndVolumeMounts { + source: stackable_operator::commons::tls_verification::TlsClientDetailsError, + }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. +/// +/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding +/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (via [`build_node_rolegroup_headless_service`] and metrics from [`build_node_rolegroup_metrics_service`]). +#[allow(clippy::too_many_arguments)] +pub fn build_server_rolegroup_statefulset( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + superset_role: &SupersetRole, + rolegroup_ref: &RoleGroupRef, + node_config: &HashMap>, + authentication_config: &SupersetClientAuthenticationDetailsResolved, + sa_name: &str, + merged_config: &SupersetConfig, +) -> Result { + let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; + let role_group = role + .role_groups + .get(&rolegroup_ref.role_group) + .context(NoNodeRoleGroupSnafu)?; + + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(recommended_object_labels.clone()) + .context(MetadataBuildSnafu)? + .build(); + + let mut pb = &mut PodBuilder::new(); + + pb = pb + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .security_context( + PodSecurityContextBuilder::new() + .fs_group(1000) // Needed for secret-operator + .build(), + ) + .affinity(&merged_config.affinity) + .service_account_name(sa_name); + + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + + for (name, value) in node_config + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default() + { + if name == SupersetConfig::CREDENTIALS_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret("SECRET_KEY", &value, "connections.secretKey"); + superset_cb.add_env_var_from_secret( + "SQLALCHEMY_DATABASE_URI", + &value, + "connections.sqlalchemyDatabaseUri", + ); + } else if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret( + "MAPBOX_API_KEY", + &value, + "connections.mapboxApiKey", + ); + } else { + superset_cb.add_env_var(name, value); + }; + } + + add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; + + let webserver_timeout = node_config + .get(&PropertyNameKind::File( + SUPERSET_CONFIG_FILENAME.to_string(), + )) + .context(MissingSupersetConfigInNodeConfigSnafu)? + .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) + .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; + + let secret = &superset.spec.cluster_config.credentials_secret; + + superset_cb + .image_from_product_image(resolved_product_image) + .add_container_port("http", APP_PORT.into()) + .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR).context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR).context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR).context(AddVolumeMountSnafu)? + .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") + .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") + .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") + .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") + .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") + // Needed by the `containerdebug` process to log it's tracing information to. + .add_env_var("CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug")) + .add_env_var("SSL_CERT_DIR", "/stackable/certs/") + .add_env_vars(authentication_env_vars(authentication_config)) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + mkdir --parents {PYTHONPATH} + cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} + cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} + + {auth_commands} + + superset db upgrade + set +x + echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' + superset fab create-admin --username \"$ADMIN_USERNAME\" --firstname \"$ADMIN_FIRSTNAME\" --lastname \"$ADMIN_LASTNAME\" --email \"$ADMIN_EMAIL\" --password \"$ADMIN_PASSWORD\" + set -x + superset init + + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + gunicorn --bind 0.0.0.0:${{SUPERSET_PORT}} --worker-class gthread --threads 20 --timeout {webserver_timeout} --limit-request-line 0 --limit-request-field_size 0 'superset.app:create_app()' & + wait_for_termination $! + + {create_vector_shutdown_file_command} + ", + auth_commands = authentication_start_commands(authentication_config), + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]) + .resources(merged_config.resources.clone().into()); + add_superset_container_probes(&mut superset_cb); + + // listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses and + // that it is possible to connect to a consistent address + let pvcs = if let Some(group_listener_name) = superset.group_listener_name(superset_role) { + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(group_listener_name), + &unversioned_recommended_labels, + ) + .build_pvc(LISTENER_VOLUME_NAME.to_owned()) + .context(BuildListenerVolumeSnafu)?; + Some(vec![pvc]) + } else { + None + }; + + superset_cb + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + + pb.add_container(superset_cb.build()); + add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; + + let metrics_container = ContainerBuilder::new("metrics") + .context(InvalidContainerNameSnafu)? + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + prepare_signal_handlers + /stackable/statsd_exporter & + wait_for_termination $! + "}]) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("64Mi") + .with_memory_limit("64Mi") + .build(), + ) + .build(); + + pb.add_volumes(controller_commons::create_volumes( + &rolegroup_ref.object_name(), + merged_config.logging.containers.get(&Container::Superset), + )) + .context(AddVolumeSnafu)?; + pb.add_container(metrics_container); + + if merged_config.logging.enable_vector_agent { + match &superset + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some(vector_aggregator_config_map_name) => { + pb.add_container( + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(role.config.pod_overrides.clone()); + pod_template.merge_from(role_group.config.pod_overrides.clone()); + + Ok(StatefulSet { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(recommended_object_labels) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("restarter.stackable.tech/enabled", "true")) + .context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(StatefulSetSpec { + // Set to `OrderedReady`, to make sure Pods start after another and the init commands don't run in parallel + pod_management_policy: Some("OrderedReady".to_string()), + replicas: role_group.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + superset, + APP_NAME, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), + template: pod_template, + volume_claim_templates: pvcs, + ..StatefulSetSpec::default() + }), + status: None, + }) +} + +fn add_superset_container_probes(superset_cb: &mut ContainerBuilder) { + let common = + ProbeBuilder::http_get_port_scheme_path(APP_PORT, None, Some("/health".to_owned())) + .with_period(Duration::from_secs(5)); + + superset_cb.startup_probe( + common + .clone() + .with_failure_threshold_duration(Duration::from_minutes_unchecked(10)) + .expect("const period is non-zero") + .build() + .expect("const duration does not overflow"), + ); + + // Remove it from the Service immediately + superset_cb.readiness_probe( + common + .clone() + .build() + .expect("const duration does not overflow"), + ); + // But only restart it after 3 failures + superset_cb.liveness_probe( + common + .with_failure_threshold(3) + .build() + .expect("const duration does not overflow"), + ); +} + +fn add_authentication_volumes_and_volume_mounts( + auth_config: &SupersetClientAuthenticationDetailsResolved, + cb: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + // Different authentication entries can reference the same secret + // class or TLS certificate. It must be ensured that the volumes + // and volume mounts are only added once in such a case. + + let mut ldap_authentication_providers = BTreeSet::new(); + let mut tls_client_credentials = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + SupersetAuthenticationClassResolved::Ldap { provider } => { + ldap_authentication_providers.insert(provider); + } + SupersetAuthenticationClassResolved::Oidc { provider, .. } => { + tls_client_credentials.insert(&provider.tls); + } + } + } + + for provider in ldap_authentication_providers { + provider + .add_volumes_and_mounts(pb, vec![cb]) + .context(AddLdapVolumesAndVolumeMountsSnafu)?; + } + + for tls in tls_client_credentials { + tls.add_volumes_and_mounts(pb, vec![cb]) + .context(AddTlsVolumesAndVolumeMountsSnafu)?; + } + + Ok(()) +} + +fn authentication_env_vars( + auth_config: &SupersetClientAuthenticationDetailsResolved, +) -> Vec { + // Different OIDC authentication entries can reference the same + // client secret. It must be ensured that the env variables are only + // added once in such a case. + + let mut oidc_client_credentials_secrets = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + SupersetAuthenticationClassResolved::Ldap { .. } => {} + SupersetAuthenticationClassResolved::Oidc { + client_auth_options: oidc, + .. + } => { + oidc_client_credentials_secrets + .insert(oidc.client_credentials_secret_ref.to_owned()); + } + } + } + + oidc_client_credentials_secrets + .iter() + .cloned() + .flat_map(stackable_operator::crd::authentication::oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts) + .collect() +} + +fn authentication_start_commands( + auth_config: &SupersetClientAuthenticationDetailsResolved, +) -> String { + let mut commands = Vec::new(); + + let mut tls_client_credentials = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + SupersetAuthenticationClassResolved::Oidc { provider, .. } => { + tls_client_credentials.insert(&provider.tls); + + // WebPKI will be handled implicitly + } + SupersetAuthenticationClassResolved::Ldap { .. } => {} + } + } + + for tls in tls_client_credentials { + commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { + add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) + })); + } + + commands + .iter() + .flatten() + .cloned() + .collect::>() + .join("\n") +} diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index cee0d7e8..e42afefd 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -1,97 +1,46 @@ //! Ensures that `Pod`s are configured and running for each [`SupersetCluster`] -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet, HashMap}, - io::Write, - sync::Arc, -}; +use std::{borrow::Cow, sync::Arc}; use const_format::concatcp; -use indoc::formatdoc; -use product_config::{ - ProductConfigManager, - flask_app_config_writer::{self, FlaskAppConfigWriterError}, - types::PropertyNameKind, -}; +use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - configmap::ConfigMapBuilder, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, - container::ContainerBuilder, - probe::ProbeBuilder, - resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, - volume::{ - ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, - ListenerReference, - }, - }, - }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ product_image_selection::{self, ResolvedProductImage}, rbac::build_rbac_resources, }, - crd::authentication::oidc, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ConfigMap, EnvVar}, - }, - apimachinery::pkg::apis::meta::v1::LabelSelector, - }, kube::{ Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Label, Labels}, logging::controller::ReconcilerError, - product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, - transform_all_roles_to_config, validate_all_roles_and_groups_config, - }, - product_logging::{ - self, - framework::{ - LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command, - }, - spec::Logging, - }, + product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::{GenericRoleConfig, RoleGroupRef}, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - utils::COMMON_BASH_TRAP_FUNCTIONS, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ OPERATOR_NAME, - authorization::opa::{OPA_IMPORTS, SupersetOpaConfigResolved}, - commands::add_cert_to_python_certifi_command, - config::{self, PYTHON_IMPORTS}, - controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + authorization::opa::SupersetOpaConfigResolved, crd::{ - APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, - SupersetConfigOptions, SupersetRole, - authentication::{ - SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, - }, - v1alpha1::{Container, SupersetCluster, SupersetClusterStatus, SupersetConfig}, + APP_NAME, SUPERSET_CONFIG_FILENAME, SupersetRole, + authentication::SupersetClientAuthenticationDetailsResolved, + v1alpha1::{SupersetCluster, SupersetClusterStatus}, + }, + operations::pdb::add_pdbs, + resources::{ + configmap::build_rolegroup_config_map, + listener::build_group_listener, + service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, + statefulset::build_server_rolegroup_statefulset, }, - listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener}, - operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::{LOG_CONFIG_FILE, extend_config_map_with_log_config}, - service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, util::build_recommended_labels, }; @@ -112,14 +61,6 @@ pub enum Error { #[snafu(display("object defines no node role"))] NoNodeRole, - #[snafu(display("object defines no node role-group"))] - NoNodeRoleGroup, - - #[snafu(display("invalid container name"))] - InvalidContainerName { - source: stackable_operator::builder::pod::container::Error, - }, - #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -136,18 +77,6 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to build config file for {rolegroup}"))] - BuildRoleGroupConfigFile { - source: FlaskAppConfigWriterError, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("failed to build ConfigMap for {rolegroup}"))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to apply ConfigMap for {rolegroup}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, @@ -170,36 +99,14 @@ pub enum Error { source: stackable_operator::product_config_utils::Error, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to apply authentication configuration"))] InvalidAuthenticationConfig { source: crate::crd::authentication::Error, }, - #[snafu(display( - "failed to get the {SUPERSET_CONFIG_FILENAME} file from node or product config" - ))] - MissingSupersetConfigInNodeConfig, - - #[snafu(display("failed to get {timeout} from {SUPERSET_CONFIG_FILENAME} file. It should be set in the product config or by user input", timeout = SupersetConfigOptions::SupersetWebserverTimeout))] - MissingWebServerTimeoutInSupersetConfig, - #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::client::Error, @@ -225,56 +132,12 @@ pub enum Error { source: crate::operations::pdb::Error, }, - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, - }, - - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to get required Labels"))] GetRequiredLabels { source: stackable_operator::kvp::KeyValuePairError, }, - #[snafu(display("failed to add Superset config settings"))] - AddSupersetConfig { source: crate::config::Error }, - - #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] - AddLdapVolumesAndVolumeMounts { - source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, - }, - - #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] - AddTlsVolumesAndVolumeMounts { - source: stackable_operator::commons::tls_verification::TlsClientDetailsError, - }, - - #[snafu(display( - "failed to write to String (Vec to be precise) containing superset config" - ))] - WriteToConfigFileString { source: std::io::Error }, - - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, - }, - #[snafu(display("SupersetCluster object is invalid"))] InvalidSupersetCluster { source: error_boundary::InvalidObject, @@ -285,26 +148,35 @@ pub enum Error { source: stackable_operator::commons::opa::Error, }, - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, - }, - #[snafu(display("failed to apply group listener"))] ApplyGroupListener { source: stackable_operator::cluster_resources::Error, }, #[snafu(display("failed to configure listener"))] - ListenerConfiguration { source: crate::listener::Error }, + ListenerConfiguration { + source: crate::resources::listener::Error, + }, #[snafu(display("failed to configure service"))] - ServiceConfiguration { source: crate::service::Error }, + ServiceConfiguration { + source: crate::resources::service::Error, + }, #[snafu(display("failed to resolve product image"))] ResolveProductImage { source: product_image_selection::Error, }, + + #[snafu(display("failed to build statefulset"))] + BuildStatefulSet { + source: crate::resources::statefulset::Error, + }, + + #[snafu(display("failed to build configmap"))] + BuildConfigMap { + source: crate::resources::configmap::Error, + }, } type Result = std::result::Result; @@ -427,7 +299,8 @@ pub async fn reconcile_superset( &auth_config, &superset_opa_config, &config.logging, - )?; + ) + .context(BuildConfigMapSnafu)?; let rg_statefulset = build_server_rolegroup_statefulset( superset, &resolved_product_image, @@ -437,7 +310,8 @@ pub async fn reconcile_superset( &auth_config, &rbac_sa.name_any(), &config, - )?; + ) + .context(BuildStatefulSetSnafu)?; let rg_metrics_service = build_node_rolegroup_metrics_service(superset, &resolved_product_image, &rolegroup) @@ -538,520 +412,6 @@ pub async fn reconcile_superset( Ok(Action::await_change()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - authentication_config: &SupersetClientAuthenticationDetailsResolved, - superset_opa_config: &Option, - logging: &Logging, -) -> Result { - let mut config_properties = BTreeMap::new(); - let mut imports = PYTHON_IMPORTS.to_vec(); - // TODO: this is true per default for versions 3.0.0 and up. - // We deactivate it here to keep existing functionality. - // However this is a security issue and should be configured properly - // Issue: https://github.com/stackabletech/superset-operator/issues/416 - config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); - - config::add_superset_config(&mut config_properties, authentication_config) - .context(AddSupersetConfigSnafu)?; - - // Adding opa configuration properties to config_properties. - // This will be injected as key/value pair in superset_config.py - if let Some(opa_config) = superset_opa_config { - // If opa role mapping is configured, insert CustomOpaSecurityManager import - imports.extend(OPA_IMPORTS); - - config_properties.extend(opa_config.as_config()); - } - - // The order here should be kept in order to preserve overrides. - // No properties should be added after this extend. - config_properties.extend( - rolegroup_config - .get(&PropertyNameKind::File( - SUPERSET_CONFIG_FILENAME.to_string(), - )) - .cloned() - .unwrap_or_default(), - ); - - let mut config_file = Vec::new(); - - // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) - if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; - } - let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - - flask_app_config_writer::write::( - &mut config_file, - config_properties.iter(), - &imports, - ) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; - - if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; - } - - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data( - SUPERSET_CONFIG_FILENAME, - String::from_utf8(config_file).unwrap(), - ); - - extend_config_map_with_log_config( - rolegroup, - logging, - &Container::Superset, - &Container::Vector, - &mut cm_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding -/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (via [`build_node_rolegroup_headless_service`] and metrics from [`build_node_rolegroup_metrics_service`]). -#[allow(clippy::too_many_arguments)] -fn build_server_rolegroup_statefulset( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, - superset_role: &SupersetRole, - rolegroup_ref: &RoleGroupRef, - node_config: &HashMap>, - authentication_config: &SupersetClientAuthenticationDetailsResolved, - sa_name: &str, - merged_config: &SupersetConfig, -) -> Result { - let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; - let role_group = role - .role_groups - .get(&rolegroup_ref.role_group) - .context(NoNodeRoleGroupSnafu)?; - - let recommended_object_labels = build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(LabelBuildSnafu)?; - - let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(recommended_object_labels.clone()) - .context(MetadataBuildSnafu)? - .build(); - - let mut pb = &mut PodBuilder::new(); - - pb = pb - .metadata(metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .security_context( - PodSecurityContextBuilder::new() - .fs_group(1000) // Needed for secret-operator - .build(), - ) - .affinity(&merged_config.affinity) - .service_account_name(sa_name); - - let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) - .context(InvalidContainerNameSnafu)?; - - for (name, value) in node_config - .get(&PropertyNameKind::Env) - .cloned() - .unwrap_or_default() - { - if name == SupersetConfig::CREDENTIALS_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret("SECRET_KEY", &value, "connections.secretKey"); - superset_cb.add_env_var_from_secret( - "SQLALCHEMY_DATABASE_URI", - &value, - "connections.sqlalchemyDatabaseUri", - ); - } else if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret( - "MAPBOX_API_KEY", - &value, - "connections.mapboxApiKey", - ); - } else { - superset_cb.add_env_var(name, value); - }; - } - - add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; - - let webserver_timeout = node_config - .get(&PropertyNameKind::File( - SUPERSET_CONFIG_FILENAME.to_string(), - )) - .context(MissingSupersetConfigInNodeConfigSnafu)? - .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) - .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; - - let secret = &superset.spec.cluster_config.credentials_secret; - - superset_cb - .image_from_product_image(resolved_product_image) - .add_container_port("http", APP_PORT.into()) - .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR).context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR).context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR).context(AddVolumeMountSnafu)? - .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") - .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") - .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") - .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") - .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") - // Needed by the `containerdebug` process to log it's tracing information to. - .add_env_var("CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug")) - .add_env_var("SSL_CERT_DIR", "/stackable/certs/") - .add_env_vars(authentication_env_vars(authentication_config)) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - - mkdir --parents {PYTHONPATH} - cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} - cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} - - {auth_commands} - - superset db upgrade - set +x - echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' - superset fab create-admin --username \"$ADMIN_USERNAME\" --firstname \"$ADMIN_FIRSTNAME\" --lastname \"$ADMIN_LASTNAME\" --email \"$ADMIN_EMAIL\" --password \"$ADMIN_PASSWORD\" - set -x - superset init - - {remove_vector_shutdown_file_command} - prepare_signal_handlers - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - gunicorn --bind 0.0.0.0:${{SUPERSET_PORT}} --worker-class gthread --threads 20 --timeout {webserver_timeout} --limit-request-line 0 --limit-request-field_size 0 'superset.app:create_app()' & - wait_for_termination $! - - {create_vector_shutdown_file_command} - ", - auth_commands = authentication_start_commands(authentication_config), - remove_vector_shutdown_file_command = - remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), - create_vector_shutdown_file_command = - create_vector_shutdown_file_command(STACKABLE_LOG_DIR), - }]) - .resources(merged_config.resources.clone().into()); - add_superset_container_probes(&mut superset_cb); - - // listener endpoints will use persistent volumes - // so that load balancers can hard-code the target addresses and - // that it is possible to connect to a consistent address - let pvcs = if let Some(group_listener_name) = superset.group_listener_name(superset_role) { - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(group_listener_name), - &unversioned_recommended_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_owned()) - .context(BuildListenerVolumeSnafu)?; - Some(vec![pvc]) - } else { - None - }; - - superset_cb - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; - - pb.add_container(superset_cb.build()); - add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; - - let metrics_container = ContainerBuilder::new("metrics") - .context(InvalidContainerNameSnafu)? - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - - prepare_signal_handlers - /stackable/statsd_exporter & - wait_for_termination $! - "}]) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("64Mi") - .with_memory_limit("64Mi") - .build(), - ) - .build(); - - pb.add_volumes(controller_commons::create_volumes( - &rolegroup_ref.object_name(), - merged_config.logging.containers.get(&Container::Superset), - )) - .context(AddVolumeSnafu)?; - pb.add_container(metrics_container); - - if merged_config.logging.enable_vector_agent { - match &superset - .spec - .cluster_config - .vector_aggregator_config_map_name - { - Some(vector_aggregator_config_map_name) => { - pb.add_container( - product_logging::framework::vector_container( - resolved_product_image, - CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - merged_config.logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } - } - - let mut pod_template = pb.build_template(); - pod_template.merge_from(role.config.pod_overrides.clone()); - pod_template.merge_from(role_group.config.pod_overrides.clone()); - - Ok(StatefulSet { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(recommended_object_labels) - .context(MetadataBuildSnafu)? - .with_label( - Label::try_from(("restarter.stackable.tech/enabled", "true")) - .context(LabelBuildSnafu)?, - ) - .build(), - spec: Some(StatefulSetSpec { - // Set to `OrderedReady`, to make sure Pods start after another and the init commands don't run in parallel - pod_management_policy: Some("OrderedReady".to_string()), - replicas: role_group.replicas.map(i32::from), - selector: LabelSelector { - match_labels: Some( - Labels::role_group_selector( - superset, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), - ..LabelSelector::default() - }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), - template: pod_template, - volume_claim_templates: pvcs, - ..StatefulSetSpec::default() - }), - status: None, - }) -} - -fn add_superset_container_probes(superset_cb: &mut ContainerBuilder) { - let common = - ProbeBuilder::http_get_port_scheme_path(APP_PORT, None, Some("/health".to_owned())) - .with_period(Duration::from_secs(5)); - - superset_cb.startup_probe( - common - .clone() - .with_failure_threshold_duration(Duration::from_minutes_unchecked(10)) - .expect("const period is non-zero") - .build() - .expect("const duration does not overflow"), - ); - - // Remove it from the Service immediately - superset_cb.readiness_probe( - common - .clone() - .build() - .expect("const duration does not overflow"), - ); - // But only restart it after 3 failures - superset_cb.liveness_probe( - common - .with_failure_threshold(3) - .build() - .expect("const duration does not overflow"), - ); -} - -fn add_authentication_volumes_and_volume_mounts( - auth_config: &SupersetClientAuthenticationDetailsResolved, - cb: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // Different authentication entries can reference the same secret - // class or TLS certificate. It must be ensured that the volumes - // and volume mounts are only added once in such a case. - - let mut ldap_authentication_providers = BTreeSet::new(); - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Ldap { provider } => { - ldap_authentication_providers.insert(provider); - } - SupersetAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - } - } - } - - for provider in ldap_authentication_providers { - provider - .add_volumes_and_mounts(pb, vec![cb]) - .context(AddLdapVolumesAndVolumeMountsSnafu)?; - } - - for tls in tls_client_credentials { - tls.add_volumes_and_mounts(pb, vec![cb]) - .context(AddTlsVolumesAndVolumeMountsSnafu)?; - } - - Ok(()) -} - -fn authentication_env_vars( - auth_config: &SupersetClientAuthenticationDetailsResolved, -) -> Vec { - // Different OIDC authentication entries can reference the same - // client secret. It must be ensured that the env variables are only - // added once in such a case. - - let mut oidc_client_credentials_secrets = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Ldap { .. } => {} - SupersetAuthenticationClassResolved::Oidc { - client_auth_options: oidc, - .. - } => { - oidc_client_credentials_secrets - .insert(oidc.client_credentials_secret_ref.to_owned()); - } - } - } - - oidc_client_credentials_secrets - .iter() - .cloned() - .flat_map(oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts) - .collect() -} - -fn authentication_start_commands( - auth_config: &SupersetClientAuthenticationDetailsResolved, -) -> String { - let mut commands = Vec::new(); - - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - - // WebPKI will be handled implicitly - } - SupersetAuthenticationClassResolved::Ldap { .. } => {} - } - } - - for tls in tls_client_credentials { - commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { - add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) - })); - } - - commands - .iter() - .flatten() - .cloned() - .collect::>() - .join("\n") -} - pub fn error_policy( _obj: Arc>, error: &Error, From 84fa755f7a5be1adc7ba65b50ea7d38406ff9510 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 26 Feb 2026 15:25:49 +0100 Subject: [PATCH 02/42] wip - cleanup --- .../src/druid_connection_controller.rs | 2 +- rust/operator-binary/src/main.rs | 1 - .../src/{util.rs => operations/job_state.rs} | 23 +------------------ rust/operator-binary/src/operations/mod.rs | 1 + .../src/resources/configmap.rs | 2 +- rust/operator-binary/src/resources/mod.rs | 23 +++++++++++++++++++ rust/operator-binary/src/resources/service.rs | 3 ++- .../src/resources/statefulset.rs | 6 +++-- .../src/superset_controller.rs | 2 +- 9 files changed, 34 insertions(+), 29 deletions(-) rename rust/operator-binary/src/{util.rs => operations/job_state.rs} (51%) diff --git a/rust/operator-binary/src/druid_connection_controller.rs b/rust/operator-binary/src/druid_connection_controller.rs index 8c6179cc..f965fef6 100644 --- a/rust/operator-binary/src/druid_connection_controller.rs +++ b/rust/operator-binary/src/druid_connection_controller.rs @@ -27,9 +27,9 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ APP_NAME, OPERATOR_NAME, crd::{PYTHONPATH, SUPERSET_CONFIG_FILENAME, druidconnection, v1alpha1}, + operations::job_state::{JobState, get_job_state}, resources::rbac, superset_controller::DOCKER_IMAGE_BASE_NAME, - util::{JobState, get_job_state}, }; pub const DRUID_CONNECTION_CONTROLLER_NAME: &str = "druid-connection"; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index d947bbc6..da56903f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -53,7 +53,6 @@ mod operations; mod product_logging; mod resources; mod superset_controller; -mod util; mod webhooks; mod built_info { diff --git a/rust/operator-binary/src/util.rs b/rust/operator-binary/src/operations/job_state.rs similarity index 51% rename from rust/operator-binary/src/util.rs rename to rust/operator-binary/src/operations/job_state.rs index 0ddf70e7..7b3defea 100644 --- a/rust/operator-binary/src/util.rs +++ b/rust/operator-binary/src/operations/job_state.rs @@ -1,6 +1,4 @@ -use stackable_operator::{k8s_openapi::api::batch::v1::Job, kvp::ObjectLabels}; - -use crate::{OPERATOR_NAME, crd::APP_NAME}; +use stackable_operator::k8s_openapi::api::batch::v1::Job; pub enum JobState { InProgress, @@ -29,22 +27,3 @@ pub fn get_job_state(job: &Job) -> JobState { JobState::InProgress } } - -/// Creates recommended `ObjectLabels` to be used in deployed resources -pub fn build_recommended_labels<'a, T>( - owner: &'a T, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -} diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs index 92ca2ec7..8da9c1c5 100644 --- a/rust/operator-binary/src/operations/mod.rs +++ b/rust/operator-binary/src/operations/mod.rs @@ -1,2 +1,3 @@ pub mod graceful_shutdown; +pub mod job_state; pub mod pdb; diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs index 2ea4f137..21e64efc 100644 --- a/rust/operator-binary/src/resources/configmap.rs +++ b/rust/operator-binary/src/resources/configmap.rs @@ -26,8 +26,8 @@ use crate::{ v1alpha1::{Container, SupersetCluster}, }, product_logging::extend_config_map_with_log_config, + resources::build_recommended_labels, superset_controller::SUPERSET_CONTROLLER_NAME, - util::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index 40970ec8..77aae153 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -1,6 +1,29 @@ +use stackable_operator::kvp::ObjectLabels; + +use crate::{OPERATOR_NAME, crd::APP_NAME}; + pub mod configmap; pub mod deployment; pub mod listener; pub mod rbac; pub mod service; pub mod statefulset; + +/// Creates recommended `ObjectLabels` to be used in deployed resources +pub fn build_recommended_labels<'a, T>( + owner: &'a T, + controller_name: &'a str, + app_version: &'a str, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, T> { + ObjectLabels { + owner, + app_name: APP_NAME, + app_version, + operator_name: OPERATOR_NAME, + controller_name, + role, + role_group, + } +} diff --git a/rust/operator-binary/src/resources/service.rs b/rust/operator-binary/src/resources/service.rs index 36dc33d2..552e7655 100644 --- a/rust/operator-binary/src/resources/service.rs +++ b/rust/operator-binary/src/resources/service.rs @@ -9,9 +9,10 @@ use stackable_operator::{ use crate::{ crd::{APP_NAME, APP_PORT, APP_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}, + resources::build_recommended_labels, superset_controller::SUPERSET_CONTROLLER_NAME, - util::build_recommended_labels, }; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("object is missing metadata to build owner reference"))] diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index f849027c..3ae9f637 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -51,9 +51,11 @@ use crate::{ }, operations::graceful_shutdown::add_graceful_shutdown_config, product_logging::LOG_CONFIG_FILE, - resources::listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, + resources::{ + build_recommended_labels, + listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, + }, superset_controller::SUPERSET_CONTROLLER_NAME, - util::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index e42afefd..0238ed7b 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -36,12 +36,12 @@ use crate::{ }, operations::pdb::add_pdbs, resources::{ + build_recommended_labels, configmap::build_rolegroup_config_map, listener::build_group_listener, service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, statefulset::build_server_rolegroup_statefulset, }, - util::build_recommended_labels, }; pub const SUPERSET_CONTROLLER_NAME: &str = "supersetcluster"; From b4597cb0c767f0b819c0e6c8a51fc3f1008ff476 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 26 Feb 2026 15:37:01 +0100 Subject: [PATCH 03/42] wip - more refactoring --- .../src/{ => config}/commands.rs | 0 rust/operator-binary/src/config/mod.rs | 3 + .../src/{ => config}/product_logging.rs | 24 +------ .../src/{config.rs => config/superset.rs} | 0 .../operator-binary/src/controller_commons.rs | 68 ------------------ rust/operator-binary/src/main.rs | 3 - .../src/resources/configmap.rs | 18 ++--- rust/operator-binary/src/resources/mod.rs | 70 ++++++++++++++++++- .../src/resources/statefulset.rs | 8 +-- 9 files changed, 80 insertions(+), 114 deletions(-) rename rust/operator-binary/src/{ => config}/commands.rs (100%) create mode 100644 rust/operator-binary/src/config/mod.rs rename rust/operator-binary/src/{ => config}/product_logging.rs (87%) rename rust/operator-binary/src/{config.rs => config/superset.rs} (100%) delete mode 100644 rust/operator-binary/src/controller_commons.rs diff --git a/rust/operator-binary/src/commands.rs b/rust/operator-binary/src/config/commands.rs similarity index 100% rename from rust/operator-binary/src/commands.rs rename to rust/operator-binary/src/config/commands.rs diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs new file mode 100644 index 00000000..c09ab723 --- /dev/null +++ b/rust/operator-binary/src/config/mod.rs @@ -0,0 +1,3 @@ +pub mod commands; +pub mod product_logging; +pub mod superset; diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/config/product_logging.rs similarity index 87% rename from rust/operator-binary/src/product_logging.rs rename to rust/operator-binary/src/config/product_logging.rs index c38a9038..166096cf 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/config/product_logging.rs @@ -1,6 +1,5 @@ use std::fmt::{Display, Write}; -use snafu::Snafu; use stackable_operator::{ builder::configmap::ConfigMapBuilder, kube::Resource, @@ -15,24 +14,6 @@ use stackable_operator::{ use crate::crd::STACKABLE_LOG_DIR; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object has no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to retrieve the ConfigMap [{cm_name}]"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - #[snafu(display("failed to retrieve the entry [{entry}] for ConfigMap [{cm_name}]"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, - }, -} - -type Result = std::result::Result; - pub const LOG_CONFIG_FILE: &str = "log_config.py"; const LOG_FILE: &str = "superset.py.json"; @@ -44,8 +25,7 @@ pub fn extend_config_map_with_log_config( main_container: &C, vector_container: &C, cm_builder: &mut ConfigMapBuilder, -) -> Result<()> -where +) where C: Clone + Ord + Display, K: Resource, { @@ -75,8 +55,6 @@ where product_logging::framework::create_vector_config(rolegroup, vector_log_config), ); } - - Ok(()) } fn create_superset_config(log_config: &AutomaticContainerLogConfig, log_dir: &str) -> String { diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config/superset.rs similarity index 100% rename from rust/operator-binary/src/config.rs rename to rust/operator-binary/src/config/superset.rs diff --git a/rust/operator-binary/src/controller_commons.rs b/rust/operator-binary/src/controller_commons.rs deleted file mode 100644 index 7d16c41b..00000000 --- a/rust/operator-binary/src/controller_commons.rs +++ /dev/null @@ -1,68 +0,0 @@ -use stackable_operator::{ - builder::pod::volume::VolumeBuilder, - k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, - product_logging::{ - self, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, -}; - -use crate::crd::MAX_LOG_FILES_SIZE; - -pub const CONFIG_VOLUME_NAME: &str = "config"; -pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; -pub const LOG_VOLUME_NAME: &str = "log"; - -pub fn create_volumes( - config_map_name: &str, - log_config: Option<&ContainerLogConfig>, -) -> Vec { - let mut volumes = Vec::new(); - - volumes.push( - VolumeBuilder::new(CONFIG_VOLUME_NAME) - .with_config_map(config_map_name) - .build(), - ); - volumes.push(Volume { - name: LOG_VOLUME_NAME.into(), - empty_dir: Some(EmptyDirVolumeSource { - medium: None, - size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( - &[MAX_LOG_FILES_SIZE], - )), - }), - ..Volume::default() - }); - - if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = log_config - { - volumes.push(Volume { - name: LOG_CONFIG_VOLUME_NAME.into(), - config_map: Some(ConfigMapVolumeSource { - name: config_map.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }); - } else { - volumes.push(Volume { - name: LOG_CONFIG_VOLUME_NAME.into(), - config_map: Some(ConfigMapVolumeSource { - name: config_map_name.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }); - } - - volumes -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index da56903f..afdb5e67 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -44,13 +44,10 @@ use crate::{ }; mod authorization; -mod commands; mod config; -mod controller_commons; mod crd; mod druid_connection_controller; mod operations; -mod product_logging; mod resources; mod superset_controller; mod webhooks; diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs index 21e64efc..61f1c46e 100644 --- a/rust/operator-binary/src/resources/configmap.rs +++ b/rust/operator-binary/src/resources/configmap.rs @@ -19,13 +19,12 @@ use stackable_operator::{ use crate::{ authorization::opa::{OPA_IMPORTS, SupersetOpaConfigResolved}, - config::{self, PYTHON_IMPORTS}, + config::{self, product_logging::extend_config_map_with_log_config, superset::PYTHON_IMPORTS}, crd::{ SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, authentication::SupersetClientAuthenticationDetailsResolved, v1alpha1::{Container, SupersetCluster}, }, - product_logging::extend_config_map_with_log_config, resources::build_recommended_labels, superset_controller::SUPERSET_CONTROLLER_NAME, }; @@ -38,7 +37,7 @@ pub enum Error { }, #[snafu(display("failed to add Superset config settings"))] - AddSupersetConfig { source: crate::config::Error }, + AddSupersetConfig { source: config::superset::Error }, #[snafu(display( "failed to write to String (Vec to be precise) containing superset config" @@ -61,12 +60,6 @@ pub enum Error { source: stackable_operator::builder::configmap::Error, rolegroup: RoleGroupRef, }, - - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, } type Result = std::result::Result; @@ -90,7 +83,7 @@ pub fn build_rolegroup_config_map( // Issue: https://github.com/stackabletech/superset-operator/issues/416 config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); - config::add_superset_config(&mut config_properties, authentication_config) + config::superset::add_superset_config(&mut config_properties, authentication_config) .context(AddSupersetConfigSnafu)?; // Adding opa configuration properties to config_properties. @@ -165,10 +158,7 @@ pub fn build_rolegroup_config_map( &Container::Superset, &Container::Vector, &mut cm_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; + ); cm_builder .build() diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index 77aae153..92b4eea5 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -1,4 +1,15 @@ -use stackable_operator::kvp::ObjectLabels; +use stackable_operator::{ + builder::pod::volume::VolumeBuilder, + k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, + kvp::ObjectLabels, + product_logging::{ + self, + spec::{ + ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, + }, + }, +}; use crate::{OPERATOR_NAME, crd::APP_NAME}; @@ -9,6 +20,12 @@ pub mod rbac; pub mod service; pub mod statefulset; +use crate::crd::MAX_LOG_FILES_SIZE; + +pub const CONFIG_VOLUME_NAME: &str = "config"; +pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; +pub const LOG_VOLUME_NAME: &str = "log"; + /// Creates recommended `ObjectLabels` to be used in deployed resources pub fn build_recommended_labels<'a, T>( owner: &'a T, @@ -27,3 +44,54 @@ pub fn build_recommended_labels<'a, T>( role_group, } } + +pub fn create_volumes( + config_map_name: &str, + log_config: Option<&ContainerLogConfig>, +) -> Vec { + let mut volumes = Vec::new(); + + volumes.push( + VolumeBuilder::new(CONFIG_VOLUME_NAME) + .with_config_map(config_map_name) + .build(), + ); + volumes.push(Volume { + name: LOG_VOLUME_NAME.into(), + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_LOG_FILES_SIZE], + )), + }), + ..Volume::default() + }); + + if let Some(ContainerLogConfig { + choice: + Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + })), + }) = log_config + { + volumes.push(Volume { + name: LOG_CONFIG_VOLUME_NAME.into(), + config_map: Some(ConfigMapVolumeSource { + name: config_map.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }); + } else { + volumes.push(Volume { + name: LOG_CONFIG_VOLUME_NAME.into(), + config_map: Some(ConfigMapVolumeSource { + name: config_map_name.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }); + } + + volumes +} diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index 3ae9f637..e71830c0 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -38,8 +38,7 @@ use stackable_operator::{ }; use crate::{ - commands::add_cert_to_python_certifi_command, - controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + config::{commands::add_cert_to_python_certifi_command, product_logging::LOG_CONFIG_FILE}, crd::{ APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, @@ -50,9 +49,8 @@ use crate::{ v1alpha1::{Container, SupersetCluster, SupersetConfig}, }, operations::graceful_shutdown::add_graceful_shutdown_config, - product_logging::LOG_CONFIG_FILE, resources::{ - build_recommended_labels, + CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME, build_recommended_labels, listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, }, superset_controller::SUPERSET_CONTROLLER_NAME, @@ -335,7 +333,7 @@ pub fn build_server_rolegroup_statefulset( ) .build(); - pb.add_volumes(controller_commons::create_volumes( + pb.add_volumes(crate::resources::create_volumes( &rolegroup_ref.object_name(), merged_config.logging.containers.get(&Container::Superset), )) From 3846bd76abeed390f7e31c2e9fad995cde23c163 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 2 Mar 2026 14:08:21 +0100 Subject: [PATCH 04/42] wip - added celery test --- .../superset-operator/templates/roles.yaml | 12 + rust/operator-binary/src/crd/mod.rs | 84 ++++- .../src/resources/deployment.rs | 346 ++++++++++++++++++ .../src/superset_controller.rs | 317 ++++++++++------ .../kuttl/celery-worker/00-limit-range.yaml | 11 + .../kuttl/celery-worker/00-patch-ns.yaml.j2 | 9 + .../kuttl/celery-worker/10-assert.yaml | 14 + .../celery-worker/10-install-postgresql.yaml | 12 + .../kuttl/celery-worker/20-assert.yaml | 22 ++ .../kuttl/celery-worker/20-install-redis.yaml | 12 + .../kuttl/celery-worker/30-assert.yaml.j2 | 10 + ...tor-aggregator-discovery-configmap.yaml.j2 | 9 + .../kuttl/celery-worker/40-assert.yaml | 56 +++ .../celery-worker/40-install-superset.yaml.j2 | 103 ++++++ .../kuttl/celery-worker/50-assert.yaml | 14 + .../celery-worker/50-install-test-pod.yaml | 35 ++ .../kuttl/celery-worker/60-assert.yaml | 9 + .../celery-worker/70-create-database.yaml | 5 + .../kuttl/celery-worker/71-run-query.yaml | 5 + .../kuttl/celery-worker/72-assert.yaml | 9 + .../kuttl/celery-worker/create-database.sh | 18 + .../helm-bitnami-postgresql-values.yaml.j2 | 44 +++ .../helm-bitnami-redis-values.yaml.j2 | 61 +++ .../kuttl/celery-worker/run-select-query.sh | 33 ++ tests/test-definition.yaml | 4 + 25 files changed, 1115 insertions(+), 139 deletions(-) create mode 100644 tests/templates/kuttl/celery-worker/00-limit-range.yaml create mode 100644 tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/celery-worker/10-assert.yaml create mode 100644 tests/templates/kuttl/celery-worker/10-install-postgresql.yaml create mode 100644 tests/templates/kuttl/celery-worker/20-assert.yaml create mode 100644 tests/templates/kuttl/celery-worker/20-install-redis.yaml create mode 100644 tests/templates/kuttl/celery-worker/30-assert.yaml.j2 create mode 100644 tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/celery-worker/40-assert.yaml create mode 100644 tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 create mode 100644 tests/templates/kuttl/celery-worker/50-assert.yaml create mode 100644 tests/templates/kuttl/celery-worker/50-install-test-pod.yaml create mode 100644 tests/templates/kuttl/celery-worker/60-assert.yaml create mode 100644 tests/templates/kuttl/celery-worker/70-create-database.yaml create mode 100644 tests/templates/kuttl/celery-worker/71-run-query.yaml create mode 100644 tests/templates/kuttl/celery-worker/72-assert.yaml create mode 100755 tests/templates/kuttl/celery-worker/create-database.sh create mode 100644 tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 create mode 100755 tests/templates/kuttl/celery-worker/run-select-query.sh diff --git a/deploy/helm/superset-operator/templates/roles.yaml b/deploy/helm/superset-operator/templates/roles.yaml index acaaafb8..a2266c96 100644 --- a/deploy/helm/superset-operator/templates/roles.yaml +++ b/deploy/helm/superset-operator/templates/roles.yaml @@ -50,6 +50,18 @@ rules: - patch - update - watch + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - create + - delete + - list + - patch + - update + - watch - apiGroups: - apps resources: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 0317420e..0036ce6b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -357,7 +357,17 @@ pub struct Connections { } #[derive( - Clone, Debug, Deserialize, Display, EnumIter, Eq, Hash, JsonSchema, PartialEq, Serialize, + Clone, + Debug, + Deserialize, + Display, + EnumIter, + EnumString, + Eq, + Hash, + JsonSchema, + PartialEq, + Serialize, )] pub enum SupersetRole { #[strum(serialize = "node")] @@ -469,24 +479,61 @@ impl v1alpha1::SupersetConfig { pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; fn default_config(cluster_name: &str, role: &SupersetRole) -> v1alpha1::SupersetConfigFragment { - // TODO: Match for roles - v1alpha1::SupersetConfigFragment { - resources: ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("300m".to_owned())), - max: Some(Quantity("1200m".to_owned())), + match role { + SupersetRole::Node => v1alpha1::SupersetConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("300m".to_owned())), + max: Some(Quantity("1200m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("2Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: v1alpha1::SupersetStorageConfigFragment {}, + }, + logging: product_logging::spec::default_logging(), + affinity: affinity::get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), + row_limit: None, + webserver_timeout: None, + }, + SupersetRole::Worker => v1alpha1::SupersetConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("1000m".to_owned())), + max: Some(Quantity("2000m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("3Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: v1alpha1::SupersetStorageConfigFragment {}, }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("2Gi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, + logging: product_logging::spec::default_logging(), + affinity: affinity::get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), + row_limit: None, + webserver_timeout: None, + }, + SupersetRole::Beat => v1alpha1::SupersetConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("200m".to_owned())), + max: Some(Quantity("500m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("1Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: v1alpha1::SupersetStorageConfigFragment {}, }, - storage: v1alpha1::SupersetStorageConfigFragment {}, + logging: product_logging::spec::default_logging(), + affinity: affinity::get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), + row_limit: None, + webserver_timeout: None, }, - logging: product_logging::spec::default_logging(), - affinity: affinity::get_affinity(cluster_name, role), - graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), - row_limit: None, - webserver_timeout: None, } } } @@ -585,13 +632,14 @@ impl v1alpha1::SupersetCluster { } /// Metadata about a node rolegroup - pub fn node_rolegroup_ref( + pub fn rolegroup_ref( &self, + role: &SupersetRole, group_name: impl Into, ) -> RoleGroupRef { RoleGroupRef { cluster: ObjectRef::from_obj(self), - role: SupersetRole::Node.to_string(), + role: role.to_string(), role_group: group_name.into(), } } diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index e69de29b..9f6888e3 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -0,0 +1,346 @@ +use std::collections::{BTreeMap, HashMap}; + +use indoc::formatdoc; +use product_config::types::PropertyNameKind; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + }, + }, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{Deployment, DeploymentSpec}, + core::v1::{ExecAction, Probe}, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kvp::{Label, Labels}, + product_logging::{ + self, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, + }, + role_utils::RoleGroupRef, + utils::COMMON_BASH_TRAP_FUNCTIONS, +}; + +use crate::{ + config::product_logging::LOG_CONFIG_FILE, + crd::{ + APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, + SupersetConfigOptions, SupersetRole, + v1alpha1::{Container, SupersetCluster, SupersetConfig}, + }, + operations::graceful_shutdown::add_graceful_shutdown_config, + resources::{ + CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME, build_recommended_labels, + }, + superset_controller::SUPERSET_CONTROLLER_NAME, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object defines no node role"))] + NoNodeRole, + + #[snafu(display("object defines no node role-group"))] + NoNodeRoleGroup, + + #[snafu(display("invalid container name"))] + InvalidContainerName { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display( + "failed to get the {SUPERSET_CONFIG_FILENAME} file from node or product config" + ))] + MissingSupersetConfigInNodeConfig, + + #[snafu(display("failed to get {timeout} from {SUPERSET_CONFIG_FILENAME} file. It should be set in the product config or by user input", timeout = SupersetConfigOptions::SupersetWebserverTimeout))] + MissingWebServerTimeoutInSupersetConfig, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { + source: product_logging::framework::LoggingError, + }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`Deployment`] runs the rolegroup, as configured by the administrator. +pub fn build_worker_rolegroup_deployment( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + superset_role: &SupersetRole, + rolegroup_ref: &RoleGroupRef, + node_config: &HashMap>, + sa_name: &str, + merged_config: &SupersetConfig, +) -> Result { + let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; + let role_group = role + .role_groups + .get(&rolegroup_ref.role_group) + .context(NoNodeRoleGroupSnafu)?; + + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(recommended_object_labels.clone()) + .context(MetadataBuildSnafu)? + .build(); + + let mut pb = &mut PodBuilder::new(); + + pb = pb + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .security_context( + PodSecurityContextBuilder::new() + .fs_group(1000) // Needed for secret-operator + .build(), + ) + .affinity(&merged_config.affinity) + .service_account_name(sa_name); + + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + + for (name, value) in node_config + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default() + { + if name == SupersetConfig::CREDENTIALS_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret("SECRET_KEY", &value, "connections.secretKey"); + superset_cb.add_env_var_from_secret( + "SQLALCHEMY_DATABASE_URI", + &value, + "connections.sqlalchemyDatabaseUri", + ); + } else if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret( + "MAPBOX_API_KEY", + &value, + "connections.mapboxApiKey", + ); + } else { + superset_cb.add_env_var(name, value); + }; + } + + let secret = &superset.spec.cluster_config.credentials_secret; + + superset_cb + .image_from_product_image(resolved_product_image) + .add_container_port("http", APP_PORT.into()) + .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") + .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") + .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") + .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") + .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") + // Needed by the `containerdebug` process to log it's tracing information to. + .add_env_var( + "CONTAINERDEBUG_LOG_DIRECTORY", + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ) + .add_env_var("SSL_CERT_DIR", "/stackable/certs/") + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + mkdir --parents {PYTHONPATH} + cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} + cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} + + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + + celery --app=superset.tasks.celery_app:app worker + + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]) + .liveness_probe(Probe { + exec: Some(ExecAction { + command: Some(vec![ + "celery -A superset.tasks.celery_app:app inspect ping -d celery@$HOSTNAME" + .to_string(), + ]), + }), + initial_delay_seconds: Some(30), + period_seconds: Some(30), + timeout_seconds: Some(30), + failure_threshold: Some(3), + ..Default::default() + }) + .resources(merged_config.resources.clone().into()); + + pb.add_container(superset_cb.build()); + add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; + + let metrics_container = ContainerBuilder::new("metrics") + .context(InvalidContainerNameSnafu)? + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + prepare_signal_handlers + /stackable/statsd_exporter & + wait_for_termination $! + "}]) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("64Mi") + .with_memory_limit("64Mi") + .build(), + ) + .build(); + + pb.add_volumes(crate::resources::create_volumes( + &rolegroup_ref.object_name(), + merged_config.logging.containers.get(&Container::Superset), + )) + .context(AddVolumeSnafu)?; + pb.add_container(metrics_container); + + if merged_config.logging.enable_vector_agent { + match &superset + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some(vector_aggregator_config_map_name) => { + pb.add_container( + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(role.config.pod_overrides.clone()); + pod_template.merge_from(role_group.config.pod_overrides.clone()); + + Ok(Deployment { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(recommended_object_labels) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("restarter.stackable.tech/enabled", "true")) + .context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(DeploymentSpec { + replicas: role_group.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + superset, + APP_NAME, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + template: pod_template, + ..DeploymentSpec::default() + }), + status: None, + }) +} diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 0238ed7b..e3a26a3e 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -1,5 +1,5 @@ //! Ensures that `Pod`s are configured and running for each [`SupersetCluster`] -use std::{borrow::Cow, sync::Arc}; +use std::{str::FromStr, sync::Arc}; use const_format::concatcp; use product_config::{ProductConfigManager, types::PropertyNameKind}; @@ -20,8 +20,8 @@ use stackable_operator::{ role_utils::{GenericRoleConfig, RoleGroupRef}, shared::time::Duration, status::condition::{ - compute_conditions, operations::ClusterOperationsConditionBuilder, - statefulset::StatefulSetConditionBuilder, + compute_conditions, deployment::DeploymentConditionBuilder, + operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -38,6 +38,7 @@ use crate::{ resources::{ build_recommended_labels, configmap::build_rolegroup_config_map, + deployment::build_worker_rolegroup_deployment, listener::build_group_listener, service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, statefulset::build_server_rolegroup_statefulset, @@ -61,6 +62,9 @@ pub enum Error { #[snafu(display("object defines no node role"))] NoNodeRole, + #[snafu(display("failed to parse role: {source}"))] + ParseRole { source: strum::ParseError }, + #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -89,6 +93,12 @@ pub enum Error { rolegroup: RoleGroupRef, }, + #[snafu(display("failed to apply Deployment for {rolegroup}"))] + ApplyRoleGroupDeployment { + source: stackable_operator::cluster_resources::Error, + rolegroup: RoleGroupRef, + }, + #[snafu(display("failed to generate product config"))] GenerateProductConfig { source: stackable_operator::product_config_utils::Error, @@ -153,30 +163,35 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to configure listener"))] - ListenerConfiguration { + #[snafu(display("failed to build listener"))] + BuildListener { source: crate::resources::listener::Error, }, - #[snafu(display("failed to configure service"))] - ServiceConfiguration { + #[snafu(display("failed to build service"))] + BuildService { source: crate::resources::service::Error, }, - #[snafu(display("failed to resolve product image"))] - ResolveProductImage { - source: product_image_selection::Error, - }, - #[snafu(display("failed to build statefulset"))] BuildStatefulSet { source: crate::resources::statefulset::Error, }, + #[snafu(display("failed to build deployment"))] + BuildDeployment { + source: crate::resources::deployment::Error, + }, + #[snafu(display("failed to build configmap"))] BuildConfigMap { source: crate::resources::configmap::Error, }, + + #[snafu(display("failed to resolve product image"))] + ResolveProductImage { + source: product_image_selection::Error, + }, } type Result = std::result::Result; @@ -205,7 +220,6 @@ pub async fn reconcile_superset( .image .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION) .context(ResolveProductImageSnafu)?; - let superset_role = SupersetRole::Node; let cluster_operation_cond_builder = ClusterOperationsConditionBuilder::new(&superset.spec.cluster_config.cluster_operation); @@ -221,16 +235,40 @@ pub async fn reconcile_superset( &resolved_product_image.product_version, &transform_all_roles_to_config( superset, - [( - superset_role.to_string(), + [ + ( + SupersetRole::Node.to_string(), + ( + vec![ + PropertyNameKind::Env, + PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), + ], + superset.spec.nodes.clone().context(NoNodeRoleSnafu)?, + ), + ), ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), - ], - superset.spec.nodes.clone().context(NoNodeRoleSnafu)?, + SupersetRole::Worker.to_string(), + ( + vec![ + PropertyNameKind::Env, + PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), + ], + // TODO: improve error + superset.spec.workers.clone().context(NoNodeRoleSnafu)?, + ), ), - )] + // ( + // SupersetRole::Beat.to_string(), + // ( + // vec![ + // PropertyNameKind::Env, + // PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), + // ], + // // TODO: improve error + // superset.spec.beat.clone().context(NoNodeRoleSnafu)?, + // ), + // ), + ] .into(), ) .context(GenerateProductConfigSnafu)?, @@ -240,11 +278,6 @@ pub async fn reconcile_superset( ) .context(InvalidProductConfigSnafu)?; - let role_node_config = validated_config - .get(superset_role.to_string().as_str()) - .map(Cow::Borrowed) - .unwrap_or_default(); - let mut cluster_resources = ClusterResources::new( APP_NAME, OPERATOR_NAME, @@ -282,117 +315,155 @@ pub async fn reconcile_superset( .await .context(ApplyRoleBindingSnafu)?; - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + let mut statefulset_cond_builder = StatefulSetConditionBuilder::default(); + let mut deployment_cond_builder = DeploymentConditionBuilder::default(); - for (rolegroup_name, rolegroup_config) in role_node_config.iter() { - let rolegroup = superset.node_rolegroup_ref(rolegroup_name); + for (superset_role_str, role_config) in validated_config { + let superset_role = SupersetRole::from_str(&superset_role_str).context(ParseRoleSnafu)?; - let config = superset - .merged_config(&SupersetRole::Node, &rolegroup) - .context(FailedToResolveConfigSnafu)?; + for (rolegroup_name, rolegroup_config) in role_config.iter() { + let rolegroup = superset.rolegroup_ref(&superset_role, rolegroup_name); - let rg_configmap = build_rolegroup_config_map( - superset, - &resolved_product_image, - &rolegroup, - rolegroup_config, - &auth_config, - &superset_opa_config, - &config.logging, - ) - .context(BuildConfigMapSnafu)?; - let rg_statefulset = build_server_rolegroup_statefulset( - superset, - &resolved_product_image, - &superset_role, - &rolegroup, - rolegroup_config, - &auth_config, - &rbac_sa.name_any(), - &config, - ) - .context(BuildStatefulSetSnafu)?; + let config = superset + .merged_config(&superset_role, &rolegroup) + .context(FailedToResolveConfigSnafu)?; - let rg_metrics_service = - build_node_rolegroup_metrics_service(superset, &resolved_product_image, &rolegroup) - .context(ServiceConfigurationSnafu)?; + let rg_configmap = build_rolegroup_config_map( + superset, + &resolved_product_image, + &rolegroup, + rolegroup_config, + &auth_config, + &superset_opa_config, + &config.logging, + ) + .context(BuildConfigMapSnafu)?; - let rg_headless_service = - build_node_rolegroup_headless_service(superset, &resolved_product_image, &rolegroup) - .context(ServiceConfigurationSnafu)?; + let rg_metrics_service = + build_node_rolegroup_metrics_service(superset, &resolved_product_image, &rolegroup) + .context(BuildServiceSnafu)?; - cluster_resources - .add(client, rg_metrics_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + let rg_headless_service = build_node_rolegroup_headless_service( + superset, + &resolved_product_image, + &rolegroup, + ) + .context(BuildServiceSnafu)?; - cluster_resources - .add(client, rg_headless_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + cluster_resources + .add(client, rg_metrics_service) + .await + .with_context(|_| ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + })?; - cluster_resources - .add(client, rg_configmap) - .await - .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - })?; - - // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts - // to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - ss_cond_builder.add( cluster_resources - .add(client, rg_statefulset.clone()) + .add(client, rg_headless_service) .await - .with_context(|_| ApplyRoleGroupStatefulSetSnafu { + .with_context(|_| ApplyRoleGroupServiceSnafu { rolegroup: rolegroup.clone(), - })?, - ); - } + })?; - if let Some(listener_class) = &superset_role.listener_class_name(superset) { - if let Some(listener_group_name) = superset.group_listener_name(&superset_role) { - let group_listener = build_group_listener( - superset, - build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.product_version, - &superset_role.to_string(), - "none", - ), - listener_class.to_string(), - listener_group_name, - ) - .context(ListenerConfigurationSnafu)?; cluster_resources - .add(client, group_listener) + .add(client, rg_configmap) + .await + .with_context(|_| ApplyRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + })?; + + match superset_role { + SupersetRole::Node => { + let rg_statefulset = build_server_rolegroup_statefulset( + superset, + &resolved_product_image, + &superset_role, + &rolegroup, + rolegroup_config, + &auth_config, + &rbac_sa.name_any(), + &config, + ) + .context(BuildStatefulSetSnafu)?; + + // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts + // to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + statefulset_cond_builder.add( + cluster_resources + .add(client, rg_statefulset.clone()) + .await + .with_context(|_| ApplyRoleGroupStatefulSetSnafu { + rolegroup: rolegroup.clone(), + })?, + ); + } + SupersetRole::Worker => { + let rg_deployment = build_worker_rolegroup_deployment( + superset, + &resolved_product_image, + &superset_role, + &rolegroup, + rolegroup_config, + &rbac_sa.name_any(), + &config, + ) + .context(BuildDeploymentSnafu)?; + + // Note: The Deployment needs to be applied after all ConfigMaps and Secrets it mounts + // to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + deployment_cond_builder.add( + cluster_resources + .add(client, rg_deployment.clone()) + .await + .with_context(|_| ApplyRoleGroupDeploymentSnafu { + rolegroup: rolegroup.clone(), + })?, + ); + } + SupersetRole::Beat => todo!(), + } + + if let Some(listener_class) = &superset_role.listener_class_name(superset) { + if let Some(listener_group_name) = superset.group_listener_name(&superset_role) { + let group_listener = build_group_listener( + superset, + build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.product_version, + &superset_role.to_string(), + "none", + ), + listener_class.to_string(), + listener_group_name, + ) + .context(BuildListenerSnafu)?; + cluster_resources + .add(client, group_listener) + .await + .context(ApplyGroupListenerSnafu)?; + } + } + + let generic_role_config = superset.generic_role_config(&superset_role); + if let Some(GenericRoleConfig { + pod_disruption_budget: pdb, + }) = generic_role_config + { + add_pdbs( + &pdb, + superset, + &superset_role, + client, + &mut cluster_resources, + ) .await - .context(ApplyGroupListenerSnafu)?; + .context(FailedToCreatePdbSnafu)?; + } } } - let generic_role_config = superset.generic_role_config(&superset_role); - if let Some(GenericRoleConfig { - pod_disruption_budget: pdb, - }) = generic_role_config - { - add_pdbs( - &pdb, - superset, - &superset_role, - client, - &mut cluster_resources, - ) - .await - .context(FailedToCreatePdbSnafu)?; - } - cluster_resources .delete_orphaned_resources(client) .await @@ -401,7 +472,11 @@ pub async fn reconcile_superset( let status = SupersetClusterStatus { conditions: compute_conditions( superset, - &[&ss_cond_builder, &cluster_operation_cond_builder], + &[ + &statefulset_cond_builder, + &deployment_cond_builder, + &cluster_operation_cond_builder, + ], ), }; client diff --git a/tests/templates/kuttl/celery-worker/00-limit-range.yaml b/tests/templates/kuttl/celery-worker/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 b/tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/celery-worker/10-assert.yaml b/tests/templates/kuttl/celery-worker/10-assert.yaml new file mode 100644 index 00000000..e9c60b15 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/10-install-postgresql.yaml b/tests/templates/kuttl/celery-worker/10-install-postgresql.yaml new file mode 100644 index 00000000..50c5ad67 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-postgresql + --namespace $NAMESPACE + --version 12.5.6 + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/celery-worker/20-assert.yaml b/tests/templates/kuttl/celery-worker/20-assert.yaml new file mode 100644 index 00000000..58d4fe84 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/20-assert.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-redis +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-redis-master +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-redis-replicas +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/20-install-redis.yaml b/tests/templates/kuttl/celery-worker/20-install-redis.yaml new file mode 100644 index 00000000..357f0fbc --- /dev/null +++ b/tests/templates/kuttl/celery-worker/20-install-redis.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-redis + --namespace $NAMESPACE + --version 17.11.3 + -f helm-bitnami-redis-values.yaml + --repo https://charts.bitnami.com/bitnami redis + --wait + timeout: 600 diff --git a/tests/templates/kuttl/celery-worker/30-assert.yaml.j2 b/tests/templates/kuttl/celery-worker/30-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/30-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/celery-worker/40-assert.yaml b/tests/templates/kuttl/celery-worker/40-assert.yaml new file mode 100644 index 00000000..e749a2d9 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/40-assert.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-superset +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-node-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-node +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-worker-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-worker +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 new file mode 100644 index 00000000..82c3c5b5 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -0,0 +1,103 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-superset +timeout: 300 +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-credentials +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Superset + adminUser.lastname: Admin + adminUser.email: admin@superset.com + adminUser.password: admin + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: superset.stackable.tech/v1alpha1 +kind: SupersetCluster +metadata: + name: superset +spec: + image: +{% if test_scenario['values']['superset'].find(",") > 0 %} + custom: "{{ test_scenario['values']['superset'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['superset'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['superset'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + credentialsSecret: superset-credentials +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + # Add endpoints that need to be exempt from CSRF protection + superset_config.py: + EXPERIMENTAL_FILE_FOOTER: |- + # Otherwise {"message": "400 Bad Request: The CSRF token is missing.", "error_type": "GENERIC_BACKEND_ERROR", "level": "error", "extra": {"issue_codes": [{"code": 1011, "message": "Issue 1011 - Superset encountered an unexpected error."}]}} + WTF_CSRF_ENABLED = False + + ### CELERY ASYNC + from flask_caching.backends.rediscache import RedisCache + RESULTS_BACKEND = RedisCache(host='superset-redis-master', port=6379, key_prefix='superset_results', password='superset') + + class CeleryConfig(object): + broker_url = "redis://:superset@superset-redis-master:6379/0" + imports = ( + "superset.sql_lab", + "superset.tasks.scheduler", + ) + result_backend = "redis://:superset@superset-redis-master:6379/0" + worker_prefetch_multiplier = 10 + task_acks_late = True + task_annotations = { + "sql_lab.get_sql_results": { + "rate_limit": "100/s", + }, + } + + CELERY_CONFIG = CeleryConfig + roleGroups: + default: + replicas: 1 + workers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + superset_config.py: + EXPERIMENTAL_FILE_FOOTER: |- + ### CELERY ASYNC + from flask_caching.backends.rediscache import RedisCache + RESULTS_BACKEND = RedisCache(host='superset-redis-master', port=6379, key_prefix='superset_results', password='superset') + + class CeleryConfig(object): + broker_url = "redis://:superset@superset-redis-master:6379/0" + imports = ( + "superset.sql_lab", + "superset.tasks.scheduler", + ) + result_backend = "redis://:superset@superset-redis-master:6379/0" + worker_prefetch_multiplier = 10 + task_acks_late = True + task_annotations = { + "sql_lab.get_sql_results": { + "rate_limit": "100/s", + }, + } + + CELERY_CONFIG = CeleryConfig + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/50-assert.yaml b/tests/templates/kuttl/celery-worker/50-assert.yaml new file mode 100644 index 00000000..f7f2887c --- /dev/null +++ b/tests/templates/kuttl/celery-worker/50-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-celery-worker +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/50-install-test-pod.yaml b/tests/templates/kuttl/celery-worker/50-install-test-pod.yaml new file mode 100644 index 00000000..3a64448f --- /dev/null +++ b/tests/templates/kuttl/celery-worker/50-install-test-pod.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-celery-worker + labels: + app: test-celery-worker +spec: + replicas: 1 + selector: + matchLabels: + app: test-celery-worker + template: + metadata: + labels: + app: test-celery-worker + spec: + containers: + - name: test-celery-worker + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + stdin: true + tty: true + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" diff --git a/tests/templates/kuttl/celery-worker/60-assert.yaml b/tests/templates/kuttl/celery-worker/60-assert.yaml new file mode 100644 index 00000000..8c58577c --- /dev/null +++ b/tests/templates/kuttl/celery-worker/60-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: copy-scripts +timeout: 300 +commands: + - script: kubectl cp -n $NAMESPACE ./create-database.sh test-celery-worker-0:/tmp + - script: kubectl cp -n $NAMESPACE ./run-select-query.sh test-celery-worker-0:/tmp diff --git a/tests/templates/kuttl/celery-worker/70-create-database.yaml b/tests/templates/kuttl/celery-worker/70-create-database.yaml new file mode 100644 index 00000000..bf902051 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/70-create-database.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl exec -n $NAMESPACE test-celery-worker-0 -- /tmp/create-database.sh diff --git a/tests/templates/kuttl/celery-worker/71-run-query.yaml b/tests/templates/kuttl/celery-worker/71-run-query.yaml new file mode 100644 index 00000000..cbd53d62 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/71-run-query.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl exec -n $NAMESPACE test-celery-worker-0 -- /tmp/run-select-query.sh diff --git a/tests/templates/kuttl/celery-worker/72-assert.yaml b/tests/templates/kuttl/celery-worker/72-assert.yaml new file mode 100644 index 00000000..05fb028e --- /dev/null +++ b/tests/templates/kuttl/celery-worker/72-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-redis +timeout: 300 +commands: + - script: kubectl exec superset-redis-master-0 --namespace $NAMESPACE -- redis-cli -h 127.0.0.1 -p 6379 --pass 'superset' KEYS '*' | grep 'superset_results' + - script: kubectl exec superset-redis-master-0 --namespace $NAMESPACE -- redis-cli -h 127.0.0.1 -p 6379 --pass 'superset' KEYS '*' | grep 'celery-task-meta-' diff --git a/tests/templates/kuttl/celery-worker/create-database.sh b/tests/templates/kuttl/celery-worker/create-database.sh new file mode 100755 index 00000000..2740f5ee --- /dev/null +++ b/tests/templates/kuttl/celery-worker/create-database.sh @@ -0,0 +1,18 @@ +#!/bin/sh +ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" \ + -H "Content-Type: application/json" \ + -d '{"username": "admin", "password": "admin", "provider": "db"}' | jq -r '.access_token') + +curl -X POST "http://superset-node:8088/api/v1/database/" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "database_name": "PostgreSQL-integrationtest", + "sqlalchemy_uri": "postgresql+psycopg2://superset:superset@superset-postgresql:5432/superset", + "expose_in_sqllab": true, + "allow_run_async": true, + "allow_ctas": false, + "allow_cvas": false, + "allow_dml": false, + "extra": "{\"allows_virtual_table_explore\": true}" + }' diff --git a/tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..2e851682 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,44 @@ +--- +global: + security: + allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images + +image: + repository: bitnamilegacy/postgresql + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + securityContext: + runAsUser: auto + +metrics: + image: + repository: bitnamilegacy/postgres-exporter + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: superset + password: superset + database: superset diff --git a/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 b/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 new file mode 100644 index 00000000..7b200ad9 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 @@ -0,0 +1,61 @@ +--- +global: + commonLabels: + stackable.tech/vendor: Stackable + security: + allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images +image: + repository: bitnamilegacy/redis +sentinel: + image: + repository: bitnamilegacy/redis-sentinel +metrics: + image: + repository: bitnamilegacy/redis-exporter +kubectl: + image: + repository: bitnamilegacy/kubectl +sysctl: + image: + repository: bitnamilegacy/os-shell + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + containerSecurityContext: + runAsUser: auto + +master: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "200m" + limits: + memory: "128Mi" + cpu: "800m" + +replica: + replicaCount: 1 + podSecurityContext: + enabled: false + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" + +auth: + password: superset diff --git a/tests/templates/kuttl/celery-worker/run-select-query.sh b/tests/templates/kuttl/celery-worker/run-select-query.sh new file mode 100755 index 00000000..c0b9b872 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/run-select-query.sh @@ -0,0 +1,33 @@ +#!/bin/sh +ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" \ + -H "Content-Type: application/json" \ + -d '{"username": "admin", "password": "admin", "provider": "db"}' | jq -r '.access_token') + +EXECUTE_QUERY_RESPONSE=$(curl -X POST "http://superset-node:8088/api/v1/sqllab/execute/" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"database_id": 1, "runAsync": true, "sql": "SELECT username from public.ab_user;"}') + +QUERY_ID=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.queryId') +QUERY_STATE=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.state') + +echo "Query started with ID: '$QUERY_ID' in state '$QUERY_STATE'" + +while [ "$QUERY_STATE" == "pending" ] || [ "$QUERY_STATE" == "running" ]; do + sleep 1 + + POLL_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/query/$QUERY_ID" \ + -H "Authorization: Bearer $ACCESS_TOKEN") + + QUERY_STATE=$(echo $POLL_RESPONSE | jq -r '.result.status') + echo "Current State: '$QUERY_STATE'" + + if [ "$QUERY_STATEE" == "failed" ]; then + echo "Query failed!" + echo $POLL_RESPONSE | jq -r '.result.error' + exit 1 + fi +done + +echo "Query finished! Fetching results..." +echo "$POLL_RESPONSE" | jq '.result' diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 2b4d9d83..da87a2cd 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -64,6 +64,10 @@ tests: dimensions: - superset - openshift + - name: celery-worker + dimensions: + - superset + - openshift suites: - name: nightly patch: From d2fc6c91a9e8ca112196ea867c828337839f8333 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 2 Mar 2026 15:15:53 +0100 Subject: [PATCH 05/42] improve test --- .../helm-bitnami-redis-values.yaml.j2 | 4 +++ .../kuttl/celery-worker/run-select-query.sh | 27 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 b/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 index 7b200ad9..28418f06 100644 --- a/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 @@ -46,7 +46,11 @@ master: replica: replicaCount: 1 podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} enabled: false +{% else %} + enabled: true +{% endif %} containerSecurityContext: enabled: false resources: diff --git a/tests/templates/kuttl/celery-worker/run-select-query.sh b/tests/templates/kuttl/celery-worker/run-select-query.sh index c0b9b872..a1653d2f 100755 --- a/tests/templates/kuttl/celery-worker/run-select-query.sh +++ b/tests/templates/kuttl/celery-worker/run-select-query.sh @@ -6,28 +6,39 @@ ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" EXECUTE_QUERY_RESPONSE=$(curl -X POST "http://superset-node:8088/api/v1/sqllab/execute/" \ -H "Authorization: Bearer $ACCESS_TOKEN" \ -H "Content-Type: application/json" \ - -d '{"database_id": 1, "runAsync": true, "sql": "SELECT username from public.ab_user;"}') + -d '{"database_id": 1, "runAsync": true, "sql": "SELECT username, first_name, last_name from public.ab_user;"}') QUERY_ID=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.queryId') QUERY_STATE=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.state') -echo "Query started with ID: '$QUERY_ID' in state '$QUERY_STATE'" +echo "Query started with ID: '$QUERY_ID' in state '$QUERY_STATE' ..." while [ "$QUERY_STATE" == "pending" ] || [ "$QUERY_STATE" == "running" ]; do - sleep 1 - POLL_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/query/$QUERY_ID" \ -H "Authorization: Bearer $ACCESS_TOKEN") QUERY_STATE=$(echo $POLL_RESPONSE | jq -r '.result.status') echo "Current State: '$QUERY_STATE'" - if [ "$QUERY_STATEE" == "failed" ]; then + if [ "$QUERY_STATE" == "failed" ]; then echo "Query failed!" - echo $POLL_RESPONSE | jq -r '.result.error' + echo "$POLL_RESPONSE" exit 1 fi + + sleep 1 done -echo "Query finished! Fetching results..." -echo "$POLL_RESPONSE" | jq '.result' +if [ "$QUERY_STATE" == "success" ]; then + RESULTS_KEY=$(echo "$POLL_RESPONSE" | jq -r '.result.results_key') + + echo "Query successful! Fetching data for results_key '$RESULTS_KEY' ..." + + DATA_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/sqllab/results/?q=%7B%0A%20%20%22key%22%3A%20%22${RESULTS_KEY}%22%0A%7D" \ + -H "Authorization: Bearer $ACCESS_TOKEN") + + echo "$DATA_RESPONSE" | jq '.data' +else + echo "Query finished with unexpected state: $QUERY_STATE" + exit 1 +fi From 2e9892c260530fd44393d1882196de297c92e102 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 2 Mar 2026 17:42:06 +0100 Subject: [PATCH 06/42] wip - improve testing --- .../src/config/product_logging.rs | 72 ++++++++++--------- .../src/resources/deployment.rs | 4 +- .../kuttl/celery-worker/create-database.sh | 10 ++- .../kuttl/celery-worker/run-select-query.sh | 12 +++- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/rust/operator-binary/src/config/product_logging.rs b/rust/operator-binary/src/config/product_logging.rs index 166096cf..3d7da1ac 100644 --- a/rust/operator-binary/src/config/product_logging.rs +++ b/rust/operator-binary/src/config/product_logging.rs @@ -1,5 +1,6 @@ use std::fmt::{Display, Write}; +use indoc::formatdoc; use stackable_operator::{ builder::configmap::ConfigMapBuilder, kube::Resource, @@ -72,41 +73,42 @@ fn create_superset_config(log_config: &AutomaticContainerLogConfig, log_dir: &st ); }); - format!( - "\ -import flask.config -import logging -import os -from superset.utils.logging_configurator import LoggingConfigurator -from pythonjsonlogger import jsonlogger - -os.makedirs('{log_dir}', exist_ok=True) - -class StackableLoggingConfigurator(LoggingConfigurator): - def configure_logging(self, app_config: flask.config.Config, debug_mode: bool): - logFormat = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' - - plainTextFormatter = logging.Formatter(logFormat) - jsonFormatter = jsonlogger.JsonFormatter(logFormat) - - consoleHandler = logging.StreamHandler() - consoleHandler.setLevel({console_log_level}) - consoleHandler.setFormatter(plainTextFormatter) - - fileHandler = logging.handlers.RotatingFileHandler( - '{log_dir}/{LOG_FILE}', - maxBytes=1048576, - backupCount=1, - ) - fileHandler.setLevel({file_log_level}) - fileHandler.setFormatter(jsonFormatter) - - rootLogger = logging.getLogger() - rootLogger.setLevel({root_log_level}) - rootLogger.addHandler(consoleHandler) - rootLogger.addHandler(fileHandler) - -{loggers_config}", + formatdoc!( + " + import flask.config + import logging + import os + from superset.utils.logging_configurator import LoggingConfigurator + from pythonjsonlogger import jsonlogger + + os.makedirs('{log_dir}', exist_ok=True) + + class StackableLoggingConfigurator(LoggingConfigurator): + def configure_logging(self, app_config: flask.config.Config, debug_mode: bool): + logFormat = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' + + plainTextFormatter = logging.Formatter(logFormat) + jsonFormatter = jsonlogger.JsonFormatter(logFormat) + + consoleHandler = logging.StreamHandler() + consoleHandler.setLevel({console_log_level}) + consoleHandler.setFormatter(plainTextFormatter) + + fileHandler = logging.handlers.RotatingFileHandler( + '{log_dir}/{LOG_FILE}', + maxBytes=1048576, + backupCount=1, + ) + fileHandler.setLevel({file_log_level}) + fileHandler.setFormatter(jsonFormatter) + + rootLogger = logging.getLogger() + rootLogger.setLevel({root_log_level}) + rootLogger.addHandler(consoleHandler) + rootLogger.addHandler(fileHandler) + + {loggers_config} + ", root_log_level = log_config.root_log_level().to_python_expression(), console_log_level = log_config .console diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 9f6888e3..72a15924 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -214,7 +214,7 @@ pub fn build_worker_rolegroup_deployment( prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - celery --app=superset.tasks.celery_app:app worker + celery --app=superset.tasks.celery_app:app worker --task-events wait_for_termination $! {create_vector_shutdown_file_command} @@ -227,7 +227,7 @@ pub fn build_worker_rolegroup_deployment( .liveness_probe(Probe { exec: Some(ExecAction { command: Some(vec![ - "celery -A superset.tasks.celery_app:app inspect ping -d celery@$HOSTNAME" + "celery --app=superset.tasks.celery_app:app inspect ping -d celery@$HOSTNAME" .to_string(), ]), }), diff --git a/tests/templates/kuttl/celery-worker/create-database.sh b/tests/templates/kuttl/celery-worker/create-database.sh index 2740f5ee..965db149 100755 --- a/tests/templates/kuttl/celery-worker/create-database.sh +++ b/tests/templates/kuttl/celery-worker/create-database.sh @@ -1,13 +1,17 @@ #!/bin/sh ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" \ - -H "Content-Type: application/json" \ - -d '{"username": "admin", "password": "admin", "provider": "db"}' | jq -r '.access_token') + -H "Content-Type: application/json" \ + -d '{ + "username": "admin", + "password": "admin", + "provider": "db" + }' | jq -r '.access_token') curl -X POST "http://superset-node:8088/api/v1/database/" \ -H "Authorization: Bearer $ACCESS_TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "database_name": "PostgreSQL-integrationtest", + "database_name": "postgresql-integrationtest", "sqlalchemy_uri": "postgresql+psycopg2://superset:superset@superset-postgresql:5432/superset", "expose_in_sqllab": true, "allow_run_async": true, diff --git a/tests/templates/kuttl/celery-worker/run-select-query.sh b/tests/templates/kuttl/celery-worker/run-select-query.sh index a1653d2f..04a450fb 100755 --- a/tests/templates/kuttl/celery-worker/run-select-query.sh +++ b/tests/templates/kuttl/celery-worker/run-select-query.sh @@ -1,12 +1,20 @@ #!/bin/sh ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" \ -H "Content-Type: application/json" \ - -d '{"username": "admin", "password": "admin", "provider": "db"}' | jq -r '.access_token') + -d '{ + "username": "admin", + "password": "admin", + "provider": "db" + }' | jq -r '.access_token') EXECUTE_QUERY_RESPONSE=$(curl -X POST "http://superset-node:8088/api/v1/sqllab/execute/" \ -H "Authorization: Bearer $ACCESS_TOKEN" \ -H "Content-Type: application/json" \ - -d '{"database_id": 1, "runAsync": true, "sql": "SELECT username, first_name, last_name from public.ab_user;"}') + -d '{ + "database_id": 1, + "runAsync": true, + "sql": "SELECT username, first_name, last_name from public.ab_user;" + }') QUERY_ID=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.queryId') QUERY_STATE=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.state') From 66c8b039d4b11bc6e8311cf2b3c85f5381dddbd4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 3 Mar 2026 12:36:12 +0100 Subject: [PATCH 07/42] add logs for worker --- .../src/resources/deployment.rs | 4 +++- .../celery-worker/40-install-superset.yaml.j2 | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 72a15924..39a69207 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -203,6 +203,8 @@ pub fn build_worker_rolegroup_deployment( "pipefail".to_string(), "-c".to_string(), ]) + // TODO: Without --loglevel=INFO, the worker does not log anyhing. + // This should be investigated and configurable. .args(vec![formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} @@ -214,7 +216,7 @@ pub fn build_worker_rolegroup_deployment( prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - celery --app=superset.tasks.celery_app:app worker --task-events + celery --app=superset.tasks.celery_app:app worker --loglevel=INFO --task-events & wait_for_termination $! {create_vector_shutdown_file_command} diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index 82c3c5b5..db1fd526 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -41,6 +41,15 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + containers: + superset: + console: + level: DEBUG + file: + level: DEBUG + loggers: + ROOT: + level: DEBUG configOverrides: # Add endpoints that need to be exempt from CSRF protection superset_config.py: @@ -75,6 +84,15 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + containers: + superset: + console: + level: DEBUG + file: + level: DEBUG + loggers: + ROOT: + level: DEBUG configOverrides: superset_config.py: EXPERIMENTAL_FILE_FOOTER: |- From 07622b4d40b951b7e968239e5b730837306a344f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 10:34:45 +0200 Subject: [PATCH 08/42] fix: remaining compile errors after merge. --- rust/operator-binary/src/crd/affinity.rs | 5 +- .../src/resources/configmap.rs | 2 +- .../src/resources/deployment.rs | 40 +- .../src/resources/statefulset.rs | 43 +- .../src/superset_controller.rs | 554 +----------------- 5 files changed, 69 insertions(+), 575 deletions(-) diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 202069d4..178746de 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -60,7 +60,10 @@ mod tests { let superset: v1alpha1::SupersetCluster = yaml_from_str_singleton_map(input).expect("illegal test input"); let merged_config = superset - .merged_config(&SupersetRole::Node, &superset.node_rolegroup_ref("default")) + .merged_config( + &SupersetRole::Node, + &superset.rolegroup_ref(&SupersetRole::Node, "default"), + ) .unwrap(); assert_eq!( diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs index 61f1c46e..09b0afaf 100644 --- a/rust/operator-binary/src/resources/configmap.rs +++ b/rust/operator-binary/src/resources/configmap.rs @@ -137,7 +137,7 @@ pub fn build_rolegroup_config_map( .name(rolegroup.object_name()) .ownerreference_from_resource(superset, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( + .with_recommended_labels(&build_recommended_labels( superset, SUPERSET_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 39a69207..9e448f51 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -12,6 +12,7 @@ use stackable_operator::{ }, }, commons::product_image_selection::ResolvedProductImage, + database_connections::TemplatingMechanism, k8s_openapi::{ DeepMerge, api::{ @@ -131,7 +132,7 @@ pub fn build_worker_rolegroup_deployment( ); let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(recommended_object_labels.clone()) + .with_recommended_labels(&recommended_object_labels) .context(MetadataBuildSnafu)? .build(); @@ -148,6 +149,16 @@ pub fn build_worker_rolegroup_deployment( .affinity(&merged_config.affinity) .service_account_name(sa_name); + // "METADATA" is the prefix for the env vars that hold the database credentials + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match + // the prefix used by the airflow-operator for consistency. + let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; + let metadata_database_connection_details = superset + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) .context(InvalidContainerNameSnafu)?; @@ -156,14 +167,7 @@ pub fn build_worker_rolegroup_deployment( .cloned() .unwrap_or_default() { - if name == SupersetConfig::CREDENTIALS_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret("SECRET_KEY", &value, "connections.secretKey"); - superset_cb.add_env_var_from_secret( - "SQLALCHEMY_DATABASE_URI", - &value, - "connections.sqlalchemyDatabaseUri", - ); - } else if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { superset_cb.add_env_var_from_secret( "MAPBOX_API_KEY", &value, @@ -174,7 +178,20 @@ pub fn build_worker_rolegroup_deployment( }; } - let secret = &superset.spec.cluster_config.credentials_secret; + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + crate::crd::INTERNAL_SECRET_SECRET_KEY, + ); + + // Database connection URL from metadataDatabase + superset_cb.add_env_var( + "SQLALCHEMY_DATABASE_URI", + metadata_database_connection_details.url_template.clone(), + ); + + let secret = &superset.spec.cluster_config.credentials_secret_name; superset_cb .image_from_product_image(resolved_product_image) @@ -241,6 +258,7 @@ pub fn build_worker_rolegroup_deployment( }) .resources(merged_config.resources.clone().into()); + metadata_database_connection_details.add_to_container(&mut superset_cb); pb.add_container(superset_cb.build()); add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; @@ -318,7 +336,7 @@ pub fn build_worker_rolegroup_deployment( .name(rolegroup_ref.object_name()) .ownerreference_from_resource(superset, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(recommended_object_labels) + .with_recommended_labels(&recommended_object_labels) .context(MetadataBuildSnafu)? .with_label( Label::try_from(("restarter.stackable.tech/enabled", "true")) diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index e71830c0..6fb46876 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -19,6 +19,7 @@ use stackable_operator::{ }, }, commons::product_image_selection::ResolvedProductImage, + database_connections::TemplatingMechanism, k8s_openapi::{ DeepMerge, api::{ @@ -162,7 +163,7 @@ pub fn build_server_rolegroup_statefulset( &rolegroup_ref.role_group, ); // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( superset, SUPERSET_CONTROLLER_NAME, // A version value is required, and we do want to use the "recommended" format for the other desired labels @@ -173,7 +174,7 @@ pub fn build_server_rolegroup_statefulset( .context(LabelBuildSnafu)?; let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(recommended_object_labels.clone()) + .with_recommended_labels(&recommended_object_labels) .context(MetadataBuildSnafu)? .build(); @@ -190,6 +191,16 @@ pub fn build_server_rolegroup_statefulset( .affinity(&merged_config.affinity) .service_account_name(sa_name); + // "METADATA" is the prefix for the env vars that hold the database credentials + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match + // the prefix used by the airflow-operator for consistency. + let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; + let metadata_database_connection_details = superset + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) .context(InvalidContainerNameSnafu)?; @@ -198,14 +209,7 @@ pub fn build_server_rolegroup_statefulset( .cloned() .unwrap_or_default() { - if name == SupersetConfig::CREDENTIALS_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret("SECRET_KEY", &value, "connections.secretKey"); - superset_cb.add_env_var_from_secret( - "SQLALCHEMY_DATABASE_URI", - &value, - "connections.sqlalchemyDatabaseUri", - ); - } else if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { superset_cb.add_env_var_from_secret( "MAPBOX_API_KEY", &value, @@ -216,6 +220,19 @@ pub fn build_server_rolegroup_statefulset( }; } + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + crate::crd::INTERNAL_SECRET_SECRET_KEY, + ); + + // Database connection URL from metadataDatabase + superset_cb.add_env_var( + "SQLALCHEMY_DATABASE_URI", + metadata_database_connection_details.url_template.clone(), + ); + add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; let webserver_timeout = node_config @@ -226,7 +243,7 @@ pub fn build_server_rolegroup_statefulset( .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; - let secret = &superset.spec.cluster_config.credentials_secret; + let secret = &superset.spec.cluster_config.credentials_secret_name; superset_cb .image_from_product_image(resolved_product_image) @@ -302,6 +319,8 @@ pub fn build_server_rolegroup_statefulset( .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)?; + metadata_database_connection_details.add_to_container(&mut superset_cb); + pb.add_container(superset_cb.build()); add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; @@ -380,7 +399,7 @@ pub fn build_server_rolegroup_statefulset( .name(rolegroup_ref.object_name()) .ownerreference_from_resource(superset, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(recommended_object_labels) + .with_recommended_labels(&recommended_object_labels) .context(MetadataBuildSnafu)? .with_label( Label::try_from(("restarter.stackable.tech/enabled", "true")) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index a0462a88..198c0405 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -11,16 +11,6 @@ use stackable_operator::{ random_secret_creation, rbac::build_rbac_resources, }, - crd::authentication::oidc, - database_connections::TemplatingMechanism, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ConfigMap, EnvVar}, - }, - apimachinery::pkg::apis::meta::v1::LabelSelector, - }, kube::{ Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, @@ -41,13 +31,9 @@ use crate::{ OPERATOR_NAME, authorization::opa::SupersetOpaConfigResolved, crd::{ - APP_NAME, APP_PORT, INTERNAL_SECRET_SECRET_KEY, METRICS_PORT, METRICS_PORT_NAME, - PYTHONPATH, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, SupersetRole, - authentication::{ - SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, - }, - v1alpha1::{Container, SupersetCluster, SupersetClusterStatus, SupersetConfig}, + APP_NAME, INTERNAL_SECRET_SECRET_KEY, SUPERSET_CONFIG_FILENAME, SupersetRole, + authentication::SupersetClientAuthenticationDetailsResolved, + v1alpha1::{SupersetCluster, SupersetClusterStatus}, }, operations::pdb::add_pdbs, resources::{ @@ -255,7 +241,7 @@ pub async fn reconcile_superset( &resolved_product_image.product_version, &transform_all_roles_to_config( superset, - [ + &[ ( SupersetRole::Node.to_string(), ( @@ -517,538 +503,6 @@ pub async fn reconcile_superset( Ok(Action::await_change()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - authentication_config: &SupersetClientAuthenticationDetailsResolved, - superset_opa_config: &Option, - logging: &Logging, -) -> Result { - let mut config_properties = BTreeMap::new(); - let mut imports = PYTHON_IMPORTS.to_vec(); - // TODO: this is true per default for versions 3.0.0 and up. - // We deactivate it here to keep existing functionality. - // However this is a security issue and should be configured properly - // Issue: https://github.com/stackabletech/superset-operator/issues/416 - config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); - - config::add_superset_config(&mut config_properties, authentication_config) - .context(AddSupersetConfigSnafu)?; - - // Adding opa configuration properties to config_properties. - // This will be injected as key/value pair in superset_config.py - if let Some(opa_config) = superset_opa_config { - // If opa role mapping is configured, insert CustomOpaSecurityManager import - imports.extend(OPA_IMPORTS); - - config_properties.extend(opa_config.as_config()); - } - - // The order here should be kept in order to preserve overrides. - // No properties should be added after this extend. - config_properties.extend( - rolegroup_config - .get(&PropertyNameKind::File( - SUPERSET_CONFIG_FILENAME.to_string(), - )) - .cloned() - .unwrap_or_default(), - ); - - let mut config_file = Vec::new(); - - // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) - if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; - } - let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - - flask_app_config_writer::write::( - &mut config_file, - config_properties.iter(), - &imports, - ) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; - - if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; - } - - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data( - SUPERSET_CONFIG_FILENAME, - String::from_utf8(config_file).unwrap(), - ); - - extend_config_map_with_log_config( - rolegroup, - logging, - &Container::Superset, - &Container::Vector, - &mut cm_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding -/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (via [`build_node_rolegroup_headless_service`] and metrics from [`build_node_rolegroup_metrics_service`]). -#[allow(clippy::too_many_arguments)] -fn build_server_rolegroup_statefulset( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, - superset_role: &SupersetRole, - rolegroup_ref: &RoleGroupRef, - node_config: &HashMap>, - authentication_config: &SupersetClientAuthenticationDetailsResolved, - sa_name: &str, - merged_config: &SupersetConfig, -) -> Result { - let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; - let role_group = role - .role_groups - .get(&rolegroup_ref.role_group) - .context(NoNodeRoleGroupSnafu)?; - - let recommended_object_labels = build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(LabelBuildSnafu)?; - - let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&recommended_object_labels) - .context(MetadataBuildSnafu)? - .build(); - - let mut pb = &mut PodBuilder::new(); - - pb = pb - .metadata(metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .security_context( - PodSecurityContextBuilder::new() - .fs_group(1000) // Needed for secret-operator - .build(), - ) - .affinity(&merged_config.affinity) - .service_account_name(sa_name); - - // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match - // the prefix used by the airflow-operator for consistency. - let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; - let metadata_database_connection_details = superset - .spec - .cluster_config - .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); - - let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) - .context(InvalidContainerNameSnafu)?; - - for (name, value) in node_config - .get(&PropertyNameKind::Env) - .cloned() - .unwrap_or_default() - { - if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret( - "MAPBOX_API_KEY", - &value, - "connections.mapboxApiKey", - ); - } else { - superset_cb.add_env_var(name, value); - }; - } - - // SECRET_KEY from auto-generated secret - superset_cb.add_env_var_from_secret( - "SECRET_KEY", - superset.shared_secret_key_secret_name(), - INTERNAL_SECRET_SECRET_KEY, - ); - - // Database connection URL from metadataDatabase - superset_cb.add_env_var( - "SQLALCHEMY_DATABASE_URI", - metadata_database_connection_details.url_template.clone(), - ); - - add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; - - let webserver_timeout = node_config - .get(&PropertyNameKind::File( - SUPERSET_CONFIG_FILENAME.to_string(), - )) - .context(MissingSupersetConfigInNodeConfigSnafu)? - .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) - .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; - - let secret = &superset.spec.cluster_config.credentials_secret_name; - - superset_cb - .image_from_product_image(resolved_product_image) - .add_container_port("http", APP_PORT.into()) - .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR).context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR).context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR).context(AddVolumeMountSnafu)? - .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") - .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") - .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") - .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") - .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") - // Needed by the `containerdebug` process to log it's tracing information to. - .add_env_var("CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug")) - .add_env_var("SSL_CERT_DIR", "/stackable/certs/") - .add_env_vars(authentication_env_vars(authentication_config)) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - - mkdir --parents {PYTHONPATH} - cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} - cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} - - {auth_commands} - - superset db upgrade - set +x - echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' - superset fab create-admin --username \"$ADMIN_USERNAME\" --firstname \"$ADMIN_FIRSTNAME\" --lastname \"$ADMIN_LASTNAME\" --email \"$ADMIN_EMAIL\" --password \"$ADMIN_PASSWORD\" - set -x - superset init - - {remove_vector_shutdown_file_command} - prepare_signal_handlers - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - gunicorn --bind 0.0.0.0:${{SUPERSET_PORT}} --worker-class gthread --threads 20 --timeout {webserver_timeout} --limit-request-line 0 --limit-request-field_size 0 'superset.app:create_app()' & - wait_for_termination $! - - {create_vector_shutdown_file_command} - ", - auth_commands = authentication_start_commands(authentication_config), - remove_vector_shutdown_file_command = - remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), - create_vector_shutdown_file_command = - create_vector_shutdown_file_command(STACKABLE_LOG_DIR), - }]) - .resources(merged_config.resources.clone().into()); - add_superset_container_probes(&mut superset_cb); - - // listener endpoints will use persistent volumes - // so that load balancers can hard-code the target addresses and - // that it is possible to connect to a consistent address - let pvcs = if let Some(group_listener_name) = superset.group_listener_name(superset_role) { - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(group_listener_name), - &unversioned_recommended_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_owned()) - .context(BuildListenerVolumeSnafu)?; - Some(vec![pvc]) - } else { - None - }; - - superset_cb - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; - - metadata_database_connection_details.add_to_container(&mut superset_cb); - - pb.add_container(superset_cb.build()); - add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; - - let metrics_container = ContainerBuilder::new("metrics") - .context(InvalidContainerNameSnafu)? - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - - prepare_signal_handlers - /stackable/statsd_exporter & - wait_for_termination $! - "}]) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("64Mi") - .with_memory_limit("64Mi") - .build(), - ) - .build(); - - pb.add_volumes(controller_commons::create_volumes( - &rolegroup_ref.object_name(), - merged_config.logging.containers.get(&Container::Superset), - )) - .context(AddVolumeSnafu)?; - pb.add_container(metrics_container); - - if merged_config.logging.enable_vector_agent { - match &superset - .spec - .cluster_config - .vector_aggregator_config_map_name - { - Some(vector_aggregator_config_map_name) => { - pb.add_container( - product_logging::framework::vector_container( - resolved_product_image, - CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - merged_config.logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } - } - - let mut pod_template = pb.build_template(); - pod_template.merge_from(role.config.pod_overrides.clone()); - pod_template.merge_from(role_group.config.pod_overrides.clone()); - - Ok(StatefulSet { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&recommended_object_labels) - .context(MetadataBuildSnafu)? - .with_label( - Label::try_from(("restarter.stackable.tech/enabled", "true")) - .context(LabelBuildSnafu)?, - ) - .build(), - spec: Some(StatefulSetSpec { - // Set to `OrderedReady`, to make sure Pods start after another and the init commands don't run in parallel - pod_management_policy: Some("OrderedReady".to_string()), - replicas: role_group.replicas.map(i32::from), - selector: LabelSelector { - match_labels: Some( - Labels::role_group_selector( - superset, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), - ..LabelSelector::default() - }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), - template: pod_template, - volume_claim_templates: pvcs, - ..StatefulSetSpec::default() - }), - status: None, - }) -} - -fn add_superset_container_probes(superset_cb: &mut ContainerBuilder) { - let common = - ProbeBuilder::http_get_port_scheme_path(APP_PORT, None, Some("/health".to_owned())) - .with_period(Duration::from_secs(5)); - - superset_cb.startup_probe( - common - .clone() - .with_failure_threshold_duration(Duration::from_minutes_unchecked(10)) - .expect("const period is non-zero") - .build() - .expect("const duration does not overflow"), - ); - - // Remove it from the Service immediately - superset_cb.readiness_probe( - common - .clone() - .build() - .expect("const duration does not overflow"), - ); - // But only restart it after 3 failures - superset_cb.liveness_probe( - common - .with_failure_threshold(3) - .build() - .expect("const duration does not overflow"), - ); -} - -fn add_authentication_volumes_and_volume_mounts( - auth_config: &SupersetClientAuthenticationDetailsResolved, - cb: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // Different authentication entries can reference the same secret - // class or TLS certificate. It must be ensured that the volumes - // and volume mounts are only added once in such a case. - - let mut ldap_authentication_providers = BTreeSet::new(); - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Ldap { provider } => { - ldap_authentication_providers.insert(provider); - } - SupersetAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - } - } - } - - for provider in ldap_authentication_providers { - provider - .add_volumes_and_mounts(pb, vec![cb]) - .context(AddLdapVolumesAndVolumeMountsSnafu)?; - } - - for tls in tls_client_credentials { - tls.add_volumes_and_mounts(pb, vec![cb]) - .context(AddTlsVolumesAndVolumeMountsSnafu)?; - } - - Ok(()) -} - -fn authentication_env_vars( - auth_config: &SupersetClientAuthenticationDetailsResolved, -) -> Vec { - // Different OIDC authentication entries can reference the same - // client secret. It must be ensured that the env variables are only - // added once in such a case. - - let mut oidc_client_credentials_secrets = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Ldap { .. } => {} - SupersetAuthenticationClassResolved::Oidc { - client_auth_options: oidc, - .. - } => { - oidc_client_credentials_secrets - .insert(oidc.client_credentials_secret_ref.to_owned()); - } - } - } - - oidc_client_credentials_secrets - .iter() - .cloned() - .flat_map(oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts) - .collect() -} - -fn authentication_start_commands( - auth_config: &SupersetClientAuthenticationDetailsResolved, -) -> String { - let mut commands = Vec::new(); - - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - - // WebPKI will be handled implicitly - } - SupersetAuthenticationClassResolved::Ldap { .. } => {} - } - } - - for tls in tls_client_credentials { - commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { - add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) - })); - } - - commands - .iter() - .flatten() - .cloned() - .collect::>() - .join("\n") -} - pub fn error_policy( _obj: Arc>, error: &Error, From b5c7bfed7335a4d7c37b36ad8dc722507c16c417 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 10:43:34 +0200 Subject: [PATCH 09/42] fix: adapt celery worker test to breaking secret changes --- .../celery-worker/40-install-superset.yaml.j2 | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index db1fd526..674d8694 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -8,7 +8,7 @@ timeout: 300 apiVersion: v1 kind: Secret metadata: - name: superset-credentials + name: superset-admin-credentials type: Opaque stringData: adminUser.username: admin @@ -16,8 +16,14 @@ stringData: adminUser.lastname: Admin adminUser.email: admin@superset.com adminUser.password: admin - connections.secretKey: thisISaSECRET_1234 - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset --- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster @@ -33,7 +39,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: superset-credentials + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From a3ce2fb856d7dcceefe40094a35d69e21f618953 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 10:43:46 +0200 Subject: [PATCH 10/42] fix: regenerated charts after merge --- extra/crds.yaml | 904 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 628 insertions(+), 276 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index e61eeee8..176e7353 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -88,64 +88,19 @@ spec: description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true properties: level: description: |- @@ -163,14 +118,141 @@ spec: nullable: true type: string type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. + description: Whether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -244,17 +326,23 @@ spec: type: integer type: object configOverrides: - additionalProperties: - additionalProperties: - type: string - type: object - default: {} description: |- The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object type: object envOverrides: additionalProperties: @@ -377,64 +465,19 @@ spec: description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true properties: level: description: |- @@ -452,14 +495,141 @@ spec: nullable: true type: string type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. + description: Whether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -533,17 +703,23 @@ spec: type: integer type: object configOverrides: - additionalProperties: - additionalProperties: - type: string - type: object - default: {} description: |- The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object type: object envOverrides: additionalProperties: @@ -1670,64 +1846,99 @@ spec: description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true properties: level: description: |- @@ -1745,14 +1956,61 @@ spec: nullable: true type: string type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. + description: Whether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -1826,17 +2084,23 @@ spec: type: integer type: object configOverrides: - additionalProperties: - additionalProperties: - type: string - type: object - default: {} description: |- The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object type: object envOverrides: additionalProperties: @@ -1959,64 +2223,19 @@ spec: description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true properties: level: description: |- @@ -2034,14 +2253,141 @@ spec: nullable: true type: string type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. + description: Whether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -2115,17 +2461,23 @@ spec: type: integer type: object configOverrides: - additionalProperties: - additionalProperties: - type: string - type: object - default: {} description: |- The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object type: object envOverrides: additionalProperties: From 0969bfb3eff42c6b220b10b4bd990ed550d733c2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 11:03:53 +0200 Subject: [PATCH 11/42] fix: adapt celery worker test to EXPERIMENTAL_FILE_FOOTER changes. --- .../templates/kuttl/celery-worker/40-install-superset.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index 674d8694..f8f80bbd 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -64,7 +64,7 @@ spec: configOverrides: # Add endpoints that need to be exempt from CSRF protection superset_config.py: - EXPERIMENTAL_FILE_FOOTER: |- + FILE_FOOTER: |- # Otherwise {"message": "400 Bad Request: The CSRF token is missing.", "error_type": "GENERIC_BACKEND_ERROR", "level": "error", "extra": {"issue_codes": [{"code": 1011, "message": "Issue 1011 - Superset encountered an unexpected error."}]}} WTF_CSRF_ENABLED = False @@ -106,7 +106,7 @@ spec: level: DEBUG configOverrides: superset_config.py: - EXPERIMENTAL_FILE_FOOTER: |- + FILE_FOOTER: |- ### CELERY ASYNC from flask_caching.backends.rediscache import RedisCache RESULTS_BACKEND = RedisCache(host='superset-redis-master', port=6379, key_prefix='superset_results', password='superset') From 756f27185f574b70c5b8e4dcb00817c2f14e1c34 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 11:46:00 +0200 Subject: [PATCH 12/42] feat: add celery backend and broker database to crd --- extra/crds.yaml | 129 ++++++++++++++++++ rust/operator-binary/src/crd/databases.rs | 49 ++++++- rust/operator-binary/src/crd/mod.rs | 21 ++- rust/operator-binary/src/resources/mod.rs | 55 +++++++- .../src/superset_controller.rs | 1 + 5 files changed, 250 insertions(+), 5 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 176e7353..b0c5023c 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -874,6 +874,135 @@ spec: required: - roleMappingFromOpa type: object + celeryBroker: + description: |- + Connection information for the celery broker queue. + + Only works if `workers` (and `beat`) roles are set. + Ignored otherwise. + nullable: true + oneOf: + - required: + - redis + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + redis: + description: |- + Connection settings for a [Redis](https://redis.io/) instance. + + Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the Redis server. + type: string + databaseId: + default: 0 + description: |- + Numeric index of the Redis logical database to use. Defaults to `0`. + + Redis supports multiple logical databases within a single instance, identified by an + integer index. Database `0` is the default. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + host: + description: Hostname or IP address of the Redis server. + type: string + port: + default: 6379 + description: Port the Redis server is listening on. Defaults to `6379`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - host + type: object + type: object + celeryResultBackend: + description: |- + Connection information for the celery backend database. + Only works if `workers` (and `beat`) roles are set. + + Ignored otherwise. + nullable: true + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + type: object clusterOperation: default: reconciliationPaused: false diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs index 5862882c..dee0785d 100644 --- a/rust/operator-binary/src/crd/databases.rs +++ b/rust/operator-binary/src/crd/databases.rs @@ -3,8 +3,11 @@ use std::ops::Deref; use serde::{Deserialize, Serialize}; use stackable_operator::{ database_connections::{ - databases::postgresql::PostgresqlConnection, - drivers::sqlalchemy::{GenericSqlAlchemyDatabaseConnection, SqlAlchemyDatabaseConnection}, + databases::{postgresql::PostgresqlConnection, redis::RedisConnection}, + drivers::{ + celery::{CeleryDatabaseConnection, GenericCeleryDatabaseConnection}, + sqlalchemy::{GenericSqlAlchemyDatabaseConnection, SqlAlchemyDatabaseConnection}, + }, }, schemars::{self, JsonSchema}, }; @@ -29,3 +32,45 @@ impl Deref for MetadataDatabaseConnection { } } } + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum CeleryResultBackendConnection { + // Docs are on the struct + Postgresql(PostgresqlConnection), + + // Docs are on the struct + Generic(GenericCeleryDatabaseConnection), +} + +impl Deref for CeleryResultBackendConnection { + type Target = dyn CeleryDatabaseConnection; + + fn deref(&self) -> &Self::Target { + match self { + Self::Postgresql(p) => p, + Self::Generic(g) => g, + } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum CeleryBrokerConnection { + // Docs are on the struct + Redis(RedisConnection), + + // Docs are on the struct + Generic(GenericCeleryDatabaseConnection), +} + +impl Deref for CeleryBrokerConnection { + type Target = dyn CeleryDatabaseConnection; + + fn deref(&self) -> &Self::Target { + match self { + Self::Redis(r) => r, + Self::Generic(g) => g, + } + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 6070ab8e..8e2b3744 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -35,7 +35,12 @@ use stackable_operator::{ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ - crd::{databases::MetadataDatabaseConnection, v1alpha1::SupersetRoleConfig}, + crd::{ + databases::{ + CeleryBrokerConnection, CeleryResultBackendConnection, MetadataDatabaseConnection, + }, + v1alpha1::SupersetRoleConfig, + }, resources::listener::default_listener_class, }; @@ -216,6 +221,20 @@ pub mod versioned { /// Configure the database where Superset stores all its internal metadata. pub metadata_database: MetadataDatabaseConnection, + /// Connection information for the celery backend database. + /// Only works if `workers` (and `beat`) roles are set. + /// + /// Ignored otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub celery_result_backend: Option, + + /// Connection information for the celery broker queue. + /// + /// Only works if `workers` (and `beat`) roles are set. + /// Ignored otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub celery_broker: Option, + /// The name of the Secret object containing the admin user credentials. /// Read the /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/superset/getting_started/first_steps) diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index 92b4eea5..a9e21c02 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -1,5 +1,12 @@ use stackable_operator::{ builder::pod::volume::VolumeBuilder, + database_connections::{ + TemplatingMechanism, + drivers::{ + celery::CeleryDatabaseConnectionDetails, + sqlalchemy::SqlAlchemyDatabaseConnectionDetails, + }, + }, k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, kvp::ObjectLabels, product_logging::{ @@ -11,7 +18,7 @@ use stackable_operator::{ }, }; -use crate::{OPERATOR_NAME, crd::APP_NAME}; +use crate::{OPERATOR_NAME, crd::APP_NAME, v1alpha1::SupersetCluster}; pub mod configmap; pub mod deployment; @@ -45,7 +52,7 @@ pub fn build_recommended_labels<'a, T>( } } -pub fn create_volumes( +pub(crate) fn create_volumes( config_map_name: &str, log_config: Option<&ContainerLogConfig>, ) -> Vec { @@ -95,3 +102,47 @@ pub fn create_volumes( volumes } + +pub(crate) fn metadata_database_connection_details( + superset: &SupersetCluster, + templating_mechanism: &TemplatingMechanism, +) -> SqlAlchemyDatabaseConnectionDetails { + superset + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism) +} + +pub(crate) fn celery_result_backend_connection_details( + superset: &SupersetCluster, + templating_mechanism: &TemplatingMechanism, +) -> Option { + // Removed the &'a and reference + superset + .spec + .cluster_config + .celery_result_backend + .as_ref() + .map(|backend| { + backend.celery_connection_details_with_templating( + "CELERY_RESULT_BACKEND", + templating_mechanism, + ) + }) +} + +pub(crate) fn celery_broker_connection_details( + superset: &SupersetCluster, + templating_mechanism: &TemplatingMechanism, +) -> Option { + // Removed the &'a and reference + superset + .spec + .cluster_config + .celery_broker + .as_ref() + .map(|broker| { + broker.celery_connection_details_with_templating("CELERY_BROKER", templating_mechanism) + }) +} diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 198c0405..c255aeee 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -263,6 +263,7 @@ pub async fn reconcile_superset( superset.spec.workers.clone().context(NoNodeRoleSnafu)?, ), ), + // TODO: uncomment // ( // SupersetRole::Beat.to_string(), // ( From b4133f895aa41d3ea710d1ebb00c660904ff161b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 15:33:56 +0200 Subject: [PATCH 13/42] feat: wire backend and broker crd values --- extra/crds.yaml | 41 ++++++------ rust/operator-binary/src/config/superset.rs | 65 +++++++++++++++--- rust/operator-binary/src/crd/databases.rs | 4 +- .../src/resources/configmap.rs | 9 ++- rust/operator-binary/src/resources/mod.rs | 15 +++-- .../src/resources/statefulset.rs | 34 +++++----- .../celery-worker/40-install-superset.yaml.j2 | 66 +++++-------------- .../kuttl/celery-worker/run-select-query.sh | 4 +- 8 files changed, 130 insertions(+), 108 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index b0c5023c..c7225333 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -948,7 +948,7 @@ spec: nullable: true oneOf: - required: - - postgresql + - redis - required: - generic properties: @@ -967,39 +967,40 @@ spec: required: - connectionUrlSecretName type: object - postgresql: - description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + redis: + description: |- + Connection settings for a [Redis](https://redis.io/) instance. + + Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). properties: credentialsSecretName: description: |- Name of a Secret containing the `username` and `password` keys used to authenticate - against the PostgreSQL server. - type: string - database: - description: Name of the database (schema) to connect to. + against the Redis server. type: string + databaseId: + default: 0 + description: |- + Numeric index of the Redis logical database to use. Defaults to `0`. + + Redis supports multiple logical databases within a single instance, identified by an + integer index. Database `0` is the default. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer host: - description: Hostname or IP address of the PostgreSQL server. + description: Hostname or IP address of the Redis server. type: string - parameters: - additionalProperties: - type: string - default: {} - description: |- - Additional map of JDBC connection parameters to append to the connection URL. The given - `HashMap` will be converted to query parameters in the form of - `?param1=value1¶m2=value2`. - type: object port: - default: 5432 - description: Port the PostgreSQL server is listening on. Defaults to `5432`. + default: 6379 + description: Port the Redis server is listening on. Defaults to `6379`. format: uint16 maximum: 65535.0 minimum: 0.0 type: integer required: - credentialsSecretName - - database - host type: object type: object diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index b94dc44a..0109c099 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -1,15 +1,22 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, io::Write}; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_operator::crd::authentication::{ldap, oidc}; -use crate::crd::{ - SupersetConfigOptions, - authentication::{ - self, DEFAULT_OIDC_PROVIDER, SupersetAuthenticationClassResolved, - SupersetClientAuthenticationDetailsResolved, +use crate::{ + crd::{ + SupersetConfigOptions, + authentication::{ + self, DEFAULT_OIDC_PROVIDER, SupersetAuthenticationClassResolved, + SupersetClientAuthenticationDetailsResolved, + }, }, + resources::{ + celery_broker_connection_details, celery_result_backend_connection_details, + metadata_database_connection_details, + }, + v1alpha1::SupersetCluster, }; #[derive(Snafu, Debug)] @@ -39,15 +46,19 @@ pub const PYTHON_IMPORTS: &[&str] = &[ pub fn add_superset_config( config: &mut BTreeMap, + superset: &SupersetCluster, authentication_config: &SupersetClientAuthenticationDetailsResolved, ) -> Result<(), Error> { + let metadata_database_url_template = + metadata_database_connection_details(superset).url_template; + config.insert( SupersetConfigOptions::SecretKey.to_string(), "os.environ.get('SECRET_KEY')".to_owned(), ); config.insert( SupersetConfigOptions::SqlalchemyDatabaseUri.to_string(), - "os.path.expandvars(os.environ.get('SQLALCHEMY_DATABASE_URI'))".to_owned(), + format!("os.path.expandvars('{metadata_database_url_template}')"), ); config.insert( SupersetConfigOptions::StatsLogger.to_string(), @@ -70,10 +81,48 @@ pub fn add_superset_config( ); append_authentication_config(config, authentication_config)?; - Ok(()) } +pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: &SupersetCluster) { + let Some(celery_result_backend_connection_details) = + celery_result_backend_connection_details(superset) + else { + return; + }; + + let Some(celery_broker_connection_details) = celery_broker_connection_details(superset) else { + return; + }; + + let result_backend_url_template = celery_result_backend_connection_details.url_template; + let broker_url_template = celery_broker_connection_details.url_template; + + // os.environ.get('{env_client_id}') + let celery_config = formatdoc!( + r#" + class CeleryConfig(object): + broker_url = os.path.expandvars('{broker_url_template}') + imports = ( + "superset.sql_lab", + "superset.tasks.scheduler", + ) + result_backend = os.path.expandvars('{result_backend_url_template}') + worker_prefetch_multiplier = 10 + task_acks_late = True + task_annotations = {{ + "sql_lab.get_sql_results": {{ + "rate_limit": "100/s", + }}, + }} + + CELERY_CONFIG = CeleryConfig + "#, + ); + + writeln!(config_file, "{celery_config}").expect("Writing to vec always works."); +} + fn append_authentication_config( config: &mut BTreeMap, auth_config: &SupersetClientAuthenticationDetailsResolved, diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs index dee0785d..8bcd3efe 100644 --- a/rust/operator-binary/src/crd/databases.rs +++ b/rust/operator-binary/src/crd/databases.rs @@ -37,7 +37,7 @@ impl Deref for MetadataDatabaseConnection { #[serde(rename_all = "camelCase")] pub enum CeleryResultBackendConnection { // Docs are on the struct - Postgresql(PostgresqlConnection), + Redis(RedisConnection), // Docs are on the struct Generic(GenericCeleryDatabaseConnection), @@ -48,7 +48,7 @@ impl Deref for CeleryResultBackendConnection { fn deref(&self) -> &Self::Target { match self { - Self::Postgresql(p) => p, + Self::Redis(r) => r, Self::Generic(g) => g, } } diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs index 09b0afaf..7c628344 100644 --- a/rust/operator-binary/src/resources/configmap.rs +++ b/rust/operator-binary/src/resources/configmap.rs @@ -83,7 +83,7 @@ pub fn build_rolegroup_config_map( // Issue: https://github.com/stackabletech/superset-operator/issues/416 config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); - config::superset::add_superset_config(&mut config_properties, authentication_config) + config::superset::add_superset_config(&mut config_properties, superset, authentication_config) .context(AddSupersetConfigSnafu)?; // Adding opa configuration properties to config_properties. @@ -111,7 +111,7 @@ pub fn build_rolegroup_config_map( // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well // (which would be bad) if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; + writeln!(config_file, "{header}").context(WriteToConfigFileStringSnafu)?; } let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); @@ -124,8 +124,11 @@ pub fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; + // We have to add a python class (no key) and cannot use the superset::config machinery. + config::superset::append_celery_worker_config(&mut config_file, superset); + if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; + writeln!(config_file, "{footer}").context(WriteToConfigFileStringSnafu)?; } let mut cm_builder = ConfigMapBuilder::new(); diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index a9e21c02..68058cce 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -105,18 +105,19 @@ pub(crate) fn create_volumes( pub(crate) fn metadata_database_connection_details( superset: &SupersetCluster, - templating_mechanism: &TemplatingMechanism, ) -> SqlAlchemyDatabaseConnectionDetails { superset .spec .cluster_config .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism) + .sqlalchemy_connection_details_with_templating( + "METADATA", + &TemplatingMechanism::BashEnvSubstitution, + ) } pub(crate) fn celery_result_backend_connection_details( superset: &SupersetCluster, - templating_mechanism: &TemplatingMechanism, ) -> Option { // Removed the &'a and reference superset @@ -127,14 +128,13 @@ pub(crate) fn celery_result_backend_connection_details( .map(|backend| { backend.celery_connection_details_with_templating( "CELERY_RESULT_BACKEND", - templating_mechanism, + &TemplatingMechanism::BashEnvSubstitution, ) }) } pub(crate) fn celery_broker_connection_details( superset: &SupersetCluster, - templating_mechanism: &TemplatingMechanism, ) -> Option { // Removed the &'a and reference superset @@ -143,6 +143,9 @@ pub(crate) fn celery_broker_connection_details( .celery_broker .as_ref() .map(|broker| { - broker.celery_connection_details_with_templating("CELERY_BROKER", templating_mechanism) + broker.celery_connection_details_with_templating( + "CELERY_BROKER", + &TemplatingMechanism::BashEnvSubstitution, + ) }) } diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index 6fb46876..5255474e 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -19,7 +19,6 @@ use stackable_operator::{ }, }, commons::product_image_selection::ResolvedProductImage, - database_connections::TemplatingMechanism, k8s_openapi::{ DeepMerge, api::{ @@ -191,18 +190,27 @@ pub fn build_server_rolegroup_statefulset( .affinity(&merged_config.affinity) .service_account_name(sa_name); + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + // "METADATA" is the prefix for the env vars that hold the database credentials // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match // the prefix used by the airflow-operator for consistency. - let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; - let metadata_database_connection_details = superset - .spec - .cluster_config - .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let metadata_database_connection_details = + super::metadata_database_connection_details(superset); + let celery_result_backend_connection_details = + super::celery_result_backend_connection_details(superset); + let celery_broker_connection_details = super::celery_broker_connection_details(superset); - let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) - .context(InvalidContainerNameSnafu)?; + metadata_database_connection_details.add_to_container(&mut superset_cb); + if let Some(celery_result_backend_connection_details) = + &celery_result_backend_connection_details + { + celery_result_backend_connection_details.add_to_container(&mut superset_cb); + } + if let Some(celery_broker_connection_details) = celery_broker_connection_details { + celery_broker_connection_details.add_to_container(&mut superset_cb); + } for (name, value) in node_config .get(&PropertyNameKind::Env) @@ -227,12 +235,6 @@ pub fn build_server_rolegroup_statefulset( crate::crd::INTERNAL_SECRET_SECRET_KEY, ); - // Database connection URL from metadataDatabase - superset_cb.add_env_var( - "SQLALCHEMY_DATABASE_URI", - metadata_database_connection_details.url_template.clone(), - ); - add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; let webserver_timeout = node_config @@ -319,8 +321,6 @@ pub fn build_server_rolegroup_statefulset( .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)?; - metadata_database_connection_details.add_to_container(&mut superset_cb); - pb.add_container(superset_cb.build()); add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index f8f80bbd..4a85e59d 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -25,6 +25,14 @@ stringData: username: superset password: superset --- +apiVersion: v1 +kind: Secret +metadata: + name: superset-redis-credentials +stringData: + username: + password: superset +--- apiVersion: superset.stackable.tech/v1alpha1 kind: SupersetCluster metadata: @@ -45,6 +53,14 @@ spec: host: superset-postgresql database: superset credentialsSecretName: superset-postgresql-credentials + celeryResultBackend: + redis: + host: superset-redis-master + credentialsSecretName: superset-redis-credentials + celeryBroker: + redis: + host: superset-redis-master + credentialsSecretName: superset-redis-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -61,33 +77,6 @@ spec: loggers: ROOT: level: DEBUG - configOverrides: - # Add endpoints that need to be exempt from CSRF protection - superset_config.py: - FILE_FOOTER: |- - # Otherwise {"message": "400 Bad Request: The CSRF token is missing.", "error_type": "GENERIC_BACKEND_ERROR", "level": "error", "extra": {"issue_codes": [{"code": 1011, "message": "Issue 1011 - Superset encountered an unexpected error."}]}} - WTF_CSRF_ENABLED = False - - ### CELERY ASYNC - from flask_caching.backends.rediscache import RedisCache - RESULTS_BACKEND = RedisCache(host='superset-redis-master', port=6379, key_prefix='superset_results', password='superset') - - class CeleryConfig(object): - broker_url = "redis://:superset@superset-redis-master:6379/0" - imports = ( - "superset.sql_lab", - "superset.tasks.scheduler", - ) - result_backend = "redis://:superset@superset-redis-master:6379/0" - worker_prefetch_multiplier = 10 - task_acks_late = True - task_annotations = { - "sql_lab.get_sql_results": { - "rate_limit": "100/s", - }, - } - - CELERY_CONFIG = CeleryConfig roleGroups: default: replicas: 1 @@ -104,29 +93,6 @@ spec: loggers: ROOT: level: DEBUG - configOverrides: - superset_config.py: - FILE_FOOTER: |- - ### CELERY ASYNC - from flask_caching.backends.rediscache import RedisCache - RESULTS_BACKEND = RedisCache(host='superset-redis-master', port=6379, key_prefix='superset_results', password='superset') - - class CeleryConfig(object): - broker_url = "redis://:superset@superset-redis-master:6379/0" - imports = ( - "superset.sql_lab", - "superset.tasks.scheduler", - ) - result_backend = "redis://:superset@superset-redis-master:6379/0" - worker_prefetch_multiplier = 10 - task_acks_late = True - task_annotations = { - "sql_lab.get_sql_results": { - "rate_limit": "100/s", - }, - } - - CELERY_CONFIG = CeleryConfig roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/run-select-query.sh b/tests/templates/kuttl/celery-worker/run-select-query.sh index 04a450fb..672cff17 100755 --- a/tests/templates/kuttl/celery-worker/run-select-query.sh +++ b/tests/templates/kuttl/celery-worker/run-select-query.sh @@ -24,7 +24,7 @@ echo "Query started with ID: '$QUERY_ID' in state '$QUERY_STATE' ..." while [ "$QUERY_STATE" == "pending" ] || [ "$QUERY_STATE" == "running" ]; do POLL_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/query/$QUERY_ID" \ -H "Authorization: Bearer $ACCESS_TOKEN") - + QUERY_STATE=$(echo $POLL_RESPONSE | jq -r '.result.status') echo "Current State: '$QUERY_STATE'" @@ -39,7 +39,7 @@ done if [ "$QUERY_STATE" == "success" ]; then RESULTS_KEY=$(echo "$POLL_RESPONSE" | jq -r '.result.results_key') - + echo "Query successful! Fetching data for results_key '$RESULTS_KEY' ..." DATA_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/sqllab/results/?q=%7B%0A%20%20%22key%22%3A%20%22${RESULTS_KEY}%22%0A%7D" \ From 586c9cbf81603de84ff3541cf8f4440cbff0a7d4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 16:52:09 +0200 Subject: [PATCH 14/42] test: celery worker test successful with beat. --- rust/operator-binary/src/config/superset.rs | 32 +- .../src/resources/deployment.rs | 290 +++++++++++++++++- .../src/superset_controller.rs | 54 +++- .../kuttl/celery-worker/40-assert.yaml | 24 ++ .../celery-worker/40-install-superset.yaml.j2 | 31 +- 5 files changed, 380 insertions(+), 51 deletions(-) diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index 0109c099..ffdc43ac 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -1,8 +1,11 @@ -use std::{collections::BTreeMap, io::Write}; +use std::{collections::BTreeMap, io::Write, str::FromStr}; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_operator::crd::authentication::{ldap, oidc}; +use stackable_operator::{ + commons::networking::{DomainName, HostName}, + crd::authentication::{ldap, oidc}, +}; use crate::{ crd::{ @@ -11,6 +14,7 @@ use crate::{ self, DEFAULT_OIDC_PROVIDER, SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, }, + databases::CeleryResultBackendConnection, }, resources::{ celery_broker_connection_details, celery_result_backend_connection_details, @@ -95,12 +99,36 @@ pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: & return; }; + // TODO: remove, unwrap hacky, only for redis. For testing. + let (celery_backend_host, celery_backend_port) = + if let Some(CeleryResultBackendConnection::Redis(redis)) = + &superset.spec.cluster_config.celery_result_backend + { + (redis.host.clone(), redis.port) + } else { + ( + HostName::DomainName(DomainName::from_str("redis").unwrap()), + 6379, + ) + }; + + let result_backend_username_env = celery_result_backend_connection_details + .username_env + .map(|env| env.name) + .unwrap_or("".to_string()); + let result_backend_password_env = celery_result_backend_connection_details + .password_env + .map(|env| env.name) + .unwrap_or("".to_string()); let result_backend_url_template = celery_result_backend_connection_details.url_template; let broker_url_template = celery_broker_connection_details.url_template; // os.environ.get('{env_client_id}') let celery_config = formatdoc!( r#" + # CELERY ASYNC + from flask_caching.backends.rediscache import RedisCache + RESULTS_BACKEND = RedisCache(host='{celery_backend_host}', port={celery_backend_port}, key_prefix='superset_results', username=os.path.expandvars('${{{result_backend_username_env}}}'), password=os.path.expandvars('${{{result_backend_password_env}}}')) class CeleryConfig(object): broker_url = os.path.expandvars('{broker_url_template}') imports = ( diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 9e448f51..3c8e8b5e 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -12,7 +12,6 @@ use stackable_operator::{ }, }, commons::product_image_selection::ResolvedProductImage, - database_connections::TemplatingMechanism, k8s_openapi::{ DeepMerge, api::{ @@ -149,18 +148,27 @@ pub fn build_worker_rolegroup_deployment( .affinity(&merged_config.affinity) .service_account_name(sa_name); + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + // "METADATA" is the prefix for the env vars that hold the database credentials // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match // the prefix used by the airflow-operator for consistency. - let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; - let metadata_database_connection_details = superset - .spec - .cluster_config - .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let metadata_database_connection_details = + super::metadata_database_connection_details(superset); + let celery_result_backend_connection_details = + super::celery_result_backend_connection_details(superset); + let celery_broker_connection_details = super::celery_broker_connection_details(superset); - let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) - .context(InvalidContainerNameSnafu)?; + metadata_database_connection_details.add_to_container(&mut superset_cb); + if let Some(celery_result_backend_connection_details) = + &celery_result_backend_connection_details + { + celery_result_backend_connection_details.add_to_container(&mut superset_cb); + } + if let Some(celery_broker_connection_details) = celery_broker_connection_details { + celery_broker_connection_details.add_to_container(&mut superset_cb); + } for (name, value) in node_config .get(&PropertyNameKind::Env) @@ -185,12 +193,6 @@ pub fn build_worker_rolegroup_deployment( crate::crd::INTERNAL_SECRET_SECRET_KEY, ); - // Database connection URL from metadataDatabase - superset_cb.add_env_var( - "SQLALCHEMY_DATABASE_URI", - metadata_database_connection_details.url_template.clone(), - ); - let secret = &superset.spec.cluster_config.credentials_secret_name; superset_cb @@ -258,7 +260,265 @@ pub fn build_worker_rolegroup_deployment( }) .resources(merged_config.resources.clone().into()); + pb.add_container(superset_cb.build()); + add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; + + let metrics_container = ContainerBuilder::new("metrics") + .context(InvalidContainerNameSnafu)? + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + prepare_signal_handlers + /stackable/statsd_exporter & + wait_for_termination $! + "}]) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("64Mi") + .with_memory_limit("64Mi") + .build(), + ) + .build(); + + pb.add_volumes(crate::resources::create_volumes( + &rolegroup_ref.object_name(), + merged_config.logging.containers.get(&Container::Superset), + )) + .context(AddVolumeSnafu)?; + pb.add_container(metrics_container); + + if merged_config.logging.enable_vector_agent { + match &superset + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some(vector_aggregator_config_map_name) => { + pb.add_container( + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(role.config.pod_overrides.clone()); + pod_template.merge_from(role_group.config.pod_overrides.clone()); + + Ok(Deployment { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("restarter.stackable.tech/enabled", "true")) + .context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(DeploymentSpec { + replicas: role_group.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + superset, + APP_NAME, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + template: pod_template, + ..DeploymentSpec::default() + }), + status: None, + }) +} + +/// The rolegroup [`Deployment`] runs the rolegroup, as configured by the administrator. +pub fn build_beat_rolegroup_deployment( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + superset_role: &SupersetRole, + rolegroup_ref: &RoleGroupRef, + node_config: &HashMap>, + sa_name: &str, + merged_config: &SupersetConfig, +) -> Result { + let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; + let role_group = role + .role_groups + .get(&rolegroup_ref.role_group) + .context(NoNodeRoleGroupSnafu)?; + + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .build(); + + let mut pb = &mut PodBuilder::new(); + + pb = pb + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .security_context( + PodSecurityContextBuilder::new() + .fs_group(1000) // Needed for secret-operator + .build(), + ) + .affinity(&merged_config.affinity) + .service_account_name(sa_name); + + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + + // "METADATA" is the prefix for the env vars that hold the database credentials + // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match + // the prefix used by the airflow-operator for consistency. + let metadata_database_connection_details = + super::metadata_database_connection_details(superset); + let celery_result_backend_connection_details = + super::celery_result_backend_connection_details(superset); + let celery_broker_connection_details = super::celery_broker_connection_details(superset); + metadata_database_connection_details.add_to_container(&mut superset_cb); + if let Some(celery_result_backend_connection_details) = + &celery_result_backend_connection_details + { + celery_result_backend_connection_details.add_to_container(&mut superset_cb); + } + if let Some(celery_broker_connection_details) = celery_broker_connection_details { + celery_broker_connection_details.add_to_container(&mut superset_cb); + } + + for (name, value) in node_config + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default() + { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret( + "MAPBOX_API_KEY", + &value, + "connections.mapboxApiKey", + ); + } else { + superset_cb.add_env_var(name, value); + }; + } + + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + crate::crd::INTERNAL_SECRET_SECRET_KEY, + ); + + let secret = &superset.spec.cluster_config.credentials_secret_name; + + superset_cb + .image_from_product_image(resolved_product_image) + .add_container_port("http", APP_PORT.into()) + .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") + .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") + .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") + .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") + .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") + // Needed by the `containerdebug` process to log it's tracing information to. + .add_env_var( + "CONTAINERDEBUG_LOG_DIRECTORY", + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ) + .add_env_var("SSL_CERT_DIR", "/stackable/certs/") + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + // TODO: Without --loglevel=INFO, the beat does not log anyhing. + // This should be investigated and configurable. + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + mkdir --parents {PYTHONPATH} + cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} + cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} + + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + + celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid --loglevel=INFO & + + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]) + .liveness_probe(Probe { + exec: Some(ExecAction { + command: Some(vec![ + "[ -f /tmp/celerybeat.pid ] && kill -0 $(cat /tmp/celerybeat.pid)".to_string(), + ]), + }), + initial_delay_seconds: Some(30), + period_seconds: Some(30), + timeout_seconds: Some(30), + failure_threshold: Some(3), + ..Default::default() + }) + .resources(merged_config.resources.clone().into()); + pb.add_container(superset_cb.build()); add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index c255aeee..b82d0af5 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -39,7 +39,7 @@ use crate::{ resources::{ build_recommended_labels, configmap::build_rolegroup_config_map, - deployment::build_worker_rolegroup_deployment, + deployment::{build_beat_rolegroup_deployment, build_worker_rolegroup_deployment}, listener::build_group_listener, service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, statefulset::build_server_rolegroup_statefulset, @@ -263,18 +263,17 @@ pub async fn reconcile_superset( superset.spec.workers.clone().context(NoNodeRoleSnafu)?, ), ), - // TODO: uncomment - // ( - // SupersetRole::Beat.to_string(), - // ( - // vec![ - // PropertyNameKind::Env, - // PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), - // ], - // // TODO: improve error - // superset.spec.beat.clone().context(NoNodeRoleSnafu)?, - // ), - // ), + ( + SupersetRole::Beat.to_string(), + ( + vec![ + PropertyNameKind::Env, + PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), + ], + // TODO: improve error + superset.spec.beat.clone().context(NoNodeRoleSnafu)?, + ), + ), ] .into(), ) @@ -415,7 +414,31 @@ pub async fn reconcile_superset( ); } SupersetRole::Worker => { - let rg_deployment = build_worker_rolegroup_deployment( + let rg_worker_deployment = build_worker_rolegroup_deployment( + superset, + &resolved_product_image, + &superset_role, + &rolegroup, + rolegroup_config, + &rbac_sa.name_any(), + &config, + ) + .context(BuildDeploymentSnafu)?; + + // Note: The Deployment needs to be applied after all ConfigMaps and Secrets it mounts + // to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + deployment_cond_builder.add( + cluster_resources + .add(client, rg_worker_deployment.clone()) + .await + .with_context(|_| ApplyRoleGroupDeploymentSnafu { + rolegroup: rolegroup.clone(), + })?, + ); + } + SupersetRole::Beat => { + let rg_beat_deployment = build_beat_rolegroup_deployment( superset, &resolved_product_image, &superset_role, @@ -431,14 +454,13 @@ pub async fn reconcile_superset( // See https://github.com/stackabletech/commons-operator/issues/111 for details. deployment_cond_builder.add( cluster_resources - .add(client, rg_deployment.clone()) + .add(client, rg_beat_deployment.clone()) .await .with_context(|_| ApplyRoleGroupDeploymentSnafu { rolegroup: rolegroup.clone(), })?, ); } - SupersetRole::Beat => todo!(), } if let Some(listener_class) = &superset_role.listener_class_name(superset) { diff --git a/tests/templates/kuttl/celery-worker/40-assert.yaml b/tests/templates/kuttl/celery-worker/40-assert.yaml index e749a2d9..f0618259 100644 --- a/tests/templates/kuttl/celery-worker/40-assert.yaml +++ b/tests/templates/kuttl/celery-worker/40-assert.yaml @@ -54,3 +54,27 @@ status: expectedPods: 1 currentHealthy: 1 disruptionsAllowed: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-beat-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-beat +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index 4a85e59d..c01bf850 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -68,15 +68,12 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - containers: - superset: - console: - level: DEBUG - file: - level: DEBUG - loggers: - ROOT: - level: DEBUG + configOverrides: + superset_config.py: + FILE_FOOTER: |- + # Add endpoints that need to be exempt from CSRF protection + # Otherwise {"message": "400 Bad Request: The CSRF token is missing.", "error_type": "GENERIC_BACKEND_ERROR", "level": "error", "extra": {"issue_codes": [{"code": 1011, "message": "Issue 1011 - Superset encountered an unexpected error."}]}} + WTF_CSRF_ENABLED = False roleGroups: default: replicas: 1 @@ -84,15 +81,13 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - containers: - superset: - console: - level: DEBUG - file: - level: DEBUG - loggers: - ROOT: - level: DEBUG + roleGroups: + default: + replicas: 1 + beat: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: replicas: 1 From 96e948efc2175bfd868085368f4297480240093c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 23 Apr 2026 18:52:58 +0200 Subject: [PATCH 15/42] fix: pre-commit --- deploy/helm/chart_testing.yaml | 1 + .../kuttl/celery-worker/40-install-superset.yaml.j2 | 2 +- tests/templates/kuttl/celery-worker/run-select-query.sh | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deploy/helm/chart_testing.yaml b/deploy/helm/chart_testing.yaml index 82b39c26..253af46d 100644 --- a/deploy/helm/chart_testing.yaml +++ b/deploy/helm/chart_testing.yaml @@ -1,3 +1,4 @@ +--- remote: origin target-branch: main chart-dirs: diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index c01bf850..33d44aa8 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -30,7 +30,7 @@ kind: Secret metadata: name: superset-redis-credentials stringData: - username: + username: password: superset --- apiVersion: superset.stackable.tech/v1alpha1 diff --git a/tests/templates/kuttl/celery-worker/run-select-query.sh b/tests/templates/kuttl/celery-worker/run-select-query.sh index 672cff17..97deda21 100755 --- a/tests/templates/kuttl/celery-worker/run-select-query.sh +++ b/tests/templates/kuttl/celery-worker/run-select-query.sh @@ -21,14 +21,14 @@ QUERY_STATE=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.state') echo "Query started with ID: '$QUERY_ID' in state '$QUERY_STATE' ..." -while [ "$QUERY_STATE" == "pending" ] || [ "$QUERY_STATE" == "running" ]; do +while [ "$QUERY_STATE" = "pending" ] || [ "$QUERY_STATE" = "running" ]; do POLL_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/query/$QUERY_ID" \ -H "Authorization: Bearer $ACCESS_TOKEN") - QUERY_STATE=$(echo $POLL_RESPONSE | jq -r '.result.status') + QUERY_STATE=$(echo "$POLL_RESPONSE" | jq -r '.result.status') echo "Current State: '$QUERY_STATE'" - if [ "$QUERY_STATE" == "failed" ]; then + if [ "$QUERY_STATE" = "failed" ]; then echo "Query failed!" echo "$POLL_RESPONSE" exit 1 @@ -37,7 +37,7 @@ while [ "$QUERY_STATE" == "pending" ] || [ "$QUERY_STATE" == "running" ]; do sleep 1 done -if [ "$QUERY_STATE" == "success" ]; then +if [ "$QUERY_STATE" = "success" ]; then RESULTS_KEY=$(echo "$POLL_RESPONSE" | jq -r '.result.results_key') echo "Query successful! Fetching data for results_key '$RESULTS_KEY' ..." From ec8dea740210c13a277c6472804a9f69de22357c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 24 Apr 2026 13:24:44 +0200 Subject: [PATCH 16/42] fix: set beat replicas to 1 or 0 only. --- rust/operator-binary/src/resources/deployment.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 3c8e8b5e..531f843e 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -604,7 +604,12 @@ pub fn build_beat_rolegroup_deployment( ) .build(), spec: Some(DeploymentSpec { - replicas: role_group.replicas.map(i32::from), + // Beat should always only be one Beat instance at a time. + // We ignore values > 1, 0 is a possible value still. + replicas: role_group + .replicas + .map(i32::from) + .map(|r| if r > 1 { 1 } else { 0 }), selector: LabelSelector { match_labels: Some( Labels::role_group_selector( From 843d1b31cf05a11c8d3568530cb5700353665eca Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 24 Apr 2026 13:39:03 +0200 Subject: [PATCH 17/42] docs: extend database connection for broker and results backend. --- .../usage-guide/database-connections.adoc | 75 ++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/docs/modules/superset/pages/usage-guide/database-connections.adoc b/docs/modules/superset/pages/usage-guide/database-connections.adoc index 355762fa..13a87608 100644 --- a/docs/modules/superset/pages/usage-guide/database-connections.adoc +++ b/docs/modules/superset/pages/usage-guide/database-connections.adoc @@ -3,8 +3,11 @@ Superset requires a metadata database for storing slices, connections, tables, dashboards and other metadata. The actual connection string is calculated by the operator so that the user does not need to remember the exact structure. +Using async queries with celery workers also requires a broker and a results backend. -== Typed connections +== Metadata database + +=== PostgreSQL [source,yaml] ---- @@ -19,7 +22,7 @@ spec: <1> A reference to one of the supported database backends (e.g. `postgresql`). <2> A reference to a Secret which must contain the two keys `username` and `password`. -== Generic connections +=== Generic connections Alternatively, these connections can also be defined in full in a referenced Secret: @@ -33,3 +36,71 @@ spec: ---- <1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `postgresql://superset:superset@superset-postgresql/superset` + +== Broker + +The broker queue is required to schedule or pass queries on the celery workers. + +=== Redis + +A redis broker can be configured in the `clusterConfig`: + +[source,yaml] +---- +spec: + clusterConfig: + celeryBroker: + redis: + host: redis-master + port: 6379 + credentialsSecretName: superset-redis-credentials +---- + +=== Generic connections + +Alternatively, these connections can also be defined in full in a referenced Secret: + +[source,yaml] +---- +spec: + clusterConfig: + celeryBroker: + generic: + connectionUrlSecretName: superset-redis-broker-url # <1> +---- + +== Results Backend + +Celery workers store their results in the configured results backend. + +=== Redis + +A redis broker can be configured in the `clusterConfig`: + +[source,yaml] +---- +spec: + clusterConfig: + celeryResultBackend: + redis: + host: redis-master + port: 6379 + credentialsSecretName: superset-redis-credentials +---- + +=== S3 + +Currently not supported. + +=== Generic connections + +Alternatively, these connections can also be defined in full in a referenced Secret: + +[source,yaml] +---- +spec: + clusterConfig: + celeryResultBackend: + generic: + connectionUrlSecretName: superset-redis-results-backend-url # <1> +---- From e9d5f6c0757e1a88e7142d80a11b43498e775648 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 24 Apr 2026 13:41:41 +0200 Subject: [PATCH 18/42] docs: fix missing footnotes. --- .../superset/pages/usage-guide/database-connections.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/modules/superset/pages/usage-guide/database-connections.adoc b/docs/modules/superset/pages/usage-guide/database-connections.adoc index 13a87608..5dc636b9 100644 --- a/docs/modules/superset/pages/usage-guide/database-connections.adoc +++ b/docs/modules/superset/pages/usage-guide/database-connections.adoc @@ -69,6 +69,8 @@ spec: connectionUrlSecretName: superset-redis-broker-url # <1> ---- +<1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `redis://:redis@redis-master/0` + == Results Backend Celery workers store their results in the configured results backend. @@ -104,3 +106,5 @@ spec: generic: connectionUrlSecretName: superset-redis-results-backend-url # <1> ---- + +<1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `redis://:redis@redis-master/0` From 69a708754a5bdf8979892f9768fd8c14c7b521f6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 24 Apr 2026 13:50:41 +0200 Subject: [PATCH 19/42] docs: add celery async query docs. --- .../usage-guide/celery-async-queries.adoc | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/modules/superset/pages/usage-guide/celery-async-queries.adoc diff --git a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc new file mode 100644 index 00000000..4539c484 --- /dev/null +++ b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc @@ -0,0 +1,36 @@ += Async Queries via Celery +:description: Schedule Apache Superset SQL queries on Celery workers. + +In order to support long running queries that exceed the web requests timeout, an asynchronous backend has to be configured. +This requires two more roles to be configured besides the existing `nodes`. The (celery) `workers` and a `beat` for scheduling. + +The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored. + +The following example shows additional required settings to enable async queries: + +[source,yaml] +---- +spec: + clusterConfig: + celeryBroker: + redis: + host: superset-redis-master + port: 6379 + credentialsSecretName: superset-redis-broker-credentials + celeryResultBackend: + redis: + host: superset-redis-master + port: 6379 + credentialsSecretName: superset-redis-results-backend-credentials + workers: + roleGroups: + default: + replicas: 2 + beat: + roleGroups: + default: + # Only 1 or 0 instances possible. + replicas: 1 +---- + +This is not a complete example. The main `nodes` role for the webserver and e.g. the metadata database are missing. From b13c2f6112f821c7c34052be919ec04c89c2d3c4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 24 Apr 2026 13:54:52 +0200 Subject: [PATCH 20/42] docs: remove outdated code comments. --- rust/operator-binary/src/resources/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index 68058cce..44f23900 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -119,7 +119,6 @@ pub(crate) fn metadata_database_connection_details( pub(crate) fn celery_result_backend_connection_details( superset: &SupersetCluster, ) -> Option { - // Removed the &'a and reference superset .spec .cluster_config @@ -136,7 +135,6 @@ pub(crate) fn celery_result_backend_connection_details( pub(crate) fn celery_broker_connection_details( superset: &SupersetCluster, ) -> Option { - // Removed the &'a and reference superset .spec .cluster_config From 60e89b3d2aa33f1f1be37f10f56720427cc39076 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 25 Apr 2026 13:40:11 +0200 Subject: [PATCH 21/42] fix: remove outdated comment --- rust/operator-binary/src/config/superset.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index ffdc43ac..a13b3204 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -123,7 +123,6 @@ pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: & let result_backend_url_template = celery_result_backend_connection_details.url_template; let broker_url_template = celery_broker_connection_details.url_template; - // os.environ.get('{env_client_id}') let celery_config = formatdoc!( r#" # CELERY ASYNC From daf088498969aeb8e5bcde33217e5df756d3eec1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 25 Apr 2026 14:48:42 +0200 Subject: [PATCH 22/42] fix: remove generic from results backend; clean up unwraps. --- extra/crds.yaml | 17 -------- rust/operator-binary/src/config/superset.rs | 31 ++++---------- rust/operator-binary/src/crd/databases.rs | 26 ++++++++++-- .../src/resources/deployment.rs | 4 +- rust/operator-binary/src/resources/mod.rs | 41 +++++++++++++------ .../src/resources/statefulset.rs | 2 +- 6 files changed, 62 insertions(+), 59 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index c7225333..8f008297 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -949,24 +949,7 @@ spec: oneOf: - required: - redis - - required: - - generic properties: - generic: - description: |- - A generic Celery database connection for broker or result backend types not covered by a - dedicated variant. - - Use this when you need a Celery-compatible connection that does not have a first-class - connection type. The complete connection URL is read from a Secret, giving the user full - control over the connection string. - properties: - connectionUrlSecretName: - description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. - type: string - required: - - connectionUrlSecretName - type: object redis: description: |- Connection settings for a [Redis](https://redis.io/) instance. diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index a13b3204..b0592665 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -1,11 +1,8 @@ -use std::{collections::BTreeMap, io::Write, str::FromStr}; +use std::{collections::BTreeMap, io::Write}; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - commons::networking::{DomainName, HostName}, - crd::authentication::{ldap, oidc}, -}; +use stackable_operator::crd::authentication::{ldap, oidc}; use crate::{ crd::{ @@ -14,7 +11,6 @@ use crate::{ self, DEFAULT_OIDC_PROVIDER, SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, }, - databases::CeleryResultBackendConnection, }, resources::{ celery_broker_connection_details, celery_result_backend_connection_details, @@ -89,8 +85,10 @@ pub fn add_superset_config( } pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: &SupersetCluster) { - let Some(celery_result_backend_connection_details) = - celery_result_backend_connection_details(superset) + let ( + Some(missing_result_backend_connection_details), + Some(celery_result_backend_connection_details), + ) = celery_result_backend_connection_details(superset) else { return; }; @@ -99,19 +97,6 @@ pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: & return; }; - // TODO: remove, unwrap hacky, only for redis. For testing. - let (celery_backend_host, celery_backend_port) = - if let Some(CeleryResultBackendConnection::Redis(redis)) = - &superset.spec.cluster_config.celery_result_backend - { - (redis.host.clone(), redis.port) - } else { - ( - HostName::DomainName(DomainName::from_str("redis").unwrap()), - 6379, - ) - }; - let result_backend_username_env = celery_result_backend_connection_details .username_env .map(|env| env.name) @@ -121,13 +106,15 @@ pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: & .map(|env| env.name) .unwrap_or("".to_string()); let result_backend_url_template = celery_result_backend_connection_details.url_template; + let result_backend_host = missing_result_backend_connection_details.host; + let result_backend_port = missing_result_backend_connection_details.port; let broker_url_template = celery_broker_connection_details.url_template; let celery_config = formatdoc!( r#" # CELERY ASYNC from flask_caching.backends.rediscache import RedisCache - RESULTS_BACKEND = RedisCache(host='{celery_backend_host}', port={celery_backend_port}, key_prefix='superset_results', username=os.path.expandvars('${{{result_backend_username_env}}}'), password=os.path.expandvars('${{{result_backend_password_env}}}')) + RESULTS_BACKEND = RedisCache(host='{result_backend_host}', port={result_backend_port}, key_prefix='superset_results', username=os.path.expandvars('${{{result_backend_username_env}}}'), password=os.path.expandvars('${{{result_backend_password_env}}}')) class CeleryConfig(object): broker_url = os.path.expandvars('{broker_url_template}') imports = ( diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs index 8bcd3efe..7104226f 100644 --- a/rust/operator-binary/src/crd/databases.rs +++ b/rust/operator-binary/src/crd/databases.rs @@ -38,9 +38,6 @@ impl Deref for MetadataDatabaseConnection { pub enum CeleryResultBackendConnection { // Docs are on the struct Redis(RedisConnection), - - // Docs are on the struct - Generic(GenericCeleryDatabaseConnection), } impl Deref for CeleryResultBackendConnection { @@ -49,11 +46,32 @@ impl Deref for CeleryResultBackendConnection { fn deref(&self) -> &Self::Target { match self { Self::Redis(r) => r, - Self::Generic(g) => g, } } } +impl CeleryResultBackendConnection { + pub fn as_python_parameters(&self) -> CeleryResultsBackendConnectionDetails { + match &self { + CeleryResultBackendConnection::Redis(redis_connection) => { + CeleryResultsBackendConnectionDetails { + host: stackable_operator::commons::networking::HostName::from( + redis_connection.host.clone(), + ), + port: redis_connection.port, + database_id: redis_connection.database_id, + } + } + } + } +} + +pub struct CeleryResultsBackendConnectionDetails { + pub host: stackable_operator::commons::networking::HostName, + pub port: u16, + pub database_id: u16, +} + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum CeleryBrokerConnection { diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 531f843e..7d2058e0 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -161,7 +161,7 @@ pub fn build_worker_rolegroup_deployment( let celery_broker_connection_details = super::celery_broker_connection_details(superset); metadata_database_connection_details.add_to_container(&mut superset_cb); - if let Some(celery_result_backend_connection_details) = + if let (_, Some(celery_result_backend_connection_details)) = &celery_result_backend_connection_details { celery_result_backend_connection_details.add_to_container(&mut superset_cb); @@ -421,7 +421,7 @@ pub fn build_beat_rolegroup_deployment( let celery_broker_connection_details = super::celery_broker_connection_details(superset); metadata_database_connection_details.add_to_container(&mut superset_cb); - if let Some(celery_result_backend_connection_details) = + if let (_, Some(celery_result_backend_connection_details)) = &celery_result_backend_connection_details { celery_result_backend_connection_details.add_to_container(&mut superset_cb); diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index 44f23900..b9f16992 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -18,7 +18,11 @@ use stackable_operator::{ }, }; -use crate::{OPERATOR_NAME, crd::APP_NAME, v1alpha1::SupersetCluster}; +use crate::{ + OPERATOR_NAME, + crd::{APP_NAME, databases::CeleryResultsBackendConnectionDetails}, + v1alpha1::SupersetCluster, +}; pub mod configmap; pub mod deployment; @@ -118,18 +122,29 @@ pub(crate) fn metadata_database_connection_details( pub(crate) fn celery_result_backend_connection_details( superset: &SupersetCluster, -) -> Option { - superset - .spec - .cluster_config - .celery_result_backend - .as_ref() - .map(|backend| { - backend.celery_connection_details_with_templating( - "CELERY_RESULT_BACKEND", - &TemplatingMechanism::BashEnvSubstitution, - ) - }) +) -> ( + Option, + Option, +) { + ( + superset + .spec + .cluster_config + .celery_result_backend + .as_ref() + .map(|backend| backend.as_python_parameters()), + superset + .spec + .cluster_config + .celery_result_backend + .as_ref() + .map(|backend| { + backend.celery_connection_details_with_templating( + "CELERY_RESULT_BACKEND", + &TemplatingMechanism::BashEnvSubstitution, + ) + }), + ) } pub(crate) fn celery_broker_connection_details( diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index 5255474e..0de4c903 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -203,7 +203,7 @@ pub fn build_server_rolegroup_statefulset( let celery_broker_connection_details = super::celery_broker_connection_details(superset); metadata_database_connection_details.add_to_container(&mut superset_cb); - if let Some(celery_result_backend_connection_details) = + if let (_, Some(celery_result_backend_connection_details)) = &celery_result_backend_connection_details { celery_result_backend_connection_details.add_to_container(&mut superset_cb); From 3d0b5cc787f9f81843c314a18931531e3be701ca Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 25 Apr 2026 15:46:40 +0200 Subject: [PATCH 23/42] fix: remove obsolete comment --- rust/operator-binary/src/resources/deployment.rs | 6 ------ rust/operator-binary/src/resources/statefulset.rs | 3 --- 2 files changed, 9 deletions(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 7d2058e0..f6cf6065 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -151,9 +151,6 @@ pub fn build_worker_rolegroup_deployment( let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) .context(InvalidContainerNameSnafu)?; - // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match - // the prefix used by the airflow-operator for consistency. let metadata_database_connection_details = super::metadata_database_connection_details(superset); let celery_result_backend_connection_details = @@ -411,9 +408,6 @@ pub fn build_beat_rolegroup_deployment( let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) .context(InvalidContainerNameSnafu)?; - // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match - // the prefix used by the airflow-operator for consistency. let metadata_database_connection_details = super::metadata_database_connection_details(superset); let celery_result_backend_connection_details = diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index 0de4c903..ec1e3182 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -193,9 +193,6 @@ pub fn build_server_rolegroup_statefulset( let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) .context(InvalidContainerNameSnafu)?; - // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match - // the prefix used by the airflow-operator for consistency. let metadata_database_connection_details = super::metadata_database_connection_details(superset); let celery_result_backend_connection_details = From 96cc92f9bc08fadec330853a4481100c7c27075a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 25 Apr 2026 15:49:45 +0200 Subject: [PATCH 24/42] fix: increase worker memory to 4GB for version 6.0.0 --- rust/operator-binary/src/crd/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 8e2b3744..63a56912 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -523,7 +523,7 @@ impl v1alpha1::SupersetConfig { max: Some(Quantity("2000m".to_owned())), }, memory: MemoryLimitsFragment { - limit: Some(Quantity("3Gi".to_owned())), + limit: Some(Quantity("4Gi".to_owned())), runtime_limits: NoRuntimeLimitsFragment {}, }, storage: v1alpha1::SupersetStorageConfigFragment {}, @@ -537,7 +537,7 @@ impl v1alpha1::SupersetConfig { SupersetRole::Beat => v1alpha1::SupersetConfigFragment { resources: ResourcesFragment { cpu: CpuLimitsFragment { - min: Some(Quantity("200m".to_owned())), + min: Some(Quantity("100m".to_owned())), max: Some(Quantity("500m".to_owned())), }, memory: MemoryLimitsFragment { From 50430eb51dbaa08feeb8ee86356be162a23ecbdb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 25 Apr 2026 15:49:54 +0200 Subject: [PATCH 25/42] fix: replicas for beat --- rust/operator-binary/src/resources/deployment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index f6cf6065..fbf2b943 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -603,7 +603,7 @@ pub fn build_beat_rolegroup_deployment( replicas: role_group .replicas .map(i32::from) - .map(|r| if r > 1 { 1 } else { 0 }), + .map(|r| if r >= 1 { 1 } else { 0 }), selector: LabelSelector { match_labels: Some( Labels::role_group_selector( From 1a56aeb9eaa34f28f31288f1a466eefe34dd2c09 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 26 Apr 2026 10:56:22 +0200 Subject: [PATCH 26/42] fix: rename result_backend_* to results_backend_* --- .../usage-guide/celery-async-queries.adoc | 4 ++-- .../usage-guide/database-connections.adoc | 4 ++-- extra/crds.yaml | 2 +- rust/operator-binary/src/config/superset.rs | 12 +++++------ rust/operator-binary/src/crd/databases.rs | 8 ++++---- rust/operator-binary/src/crd/mod.rs | 4 ++-- .../src/resources/deployment.rs | 20 +++++++++---------- rust/operator-binary/src/resources/mod.rs | 8 ++++---- .../src/resources/statefulset.rs | 10 +++++----- .../celery-worker/40-install-superset.yaml.j2 | 2 +- 10 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc index 4539c484..e6d1775e 100644 --- a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc +++ b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc @@ -4,7 +4,7 @@ In order to support long running queries that exceed the web requests timeout, an asynchronous backend has to be configured. This requires two more roles to be configured besides the existing `nodes`. The (celery) `workers` and a `beat` for scheduling. -The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored. +The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored and set to `1`. The following example shows additional required settings to enable async queries: @@ -17,7 +17,7 @@ spec: host: superset-redis-master port: 6379 credentialsSecretName: superset-redis-broker-credentials - celeryResultBackend: + celeryResultsBackend: redis: host: superset-redis-master port: 6379 diff --git a/docs/modules/superset/pages/usage-guide/database-connections.adoc b/docs/modules/superset/pages/usage-guide/database-connections.adoc index 5dc636b9..06c12827 100644 --- a/docs/modules/superset/pages/usage-guide/database-connections.adoc +++ b/docs/modules/superset/pages/usage-guide/database-connections.adoc @@ -83,7 +83,7 @@ A redis broker can be configured in the `clusterConfig`: ---- spec: clusterConfig: - celeryResultBackend: + celeryResultsBackend: redis: host: redis-master port: 6379 @@ -102,7 +102,7 @@ Alternatively, these connections can also be defined in full in a referenced Sec ---- spec: clusterConfig: - celeryResultBackend: + celeryResultsBackend: generic: connectionUrlSecretName: superset-redis-results-backend-url # <1> ---- diff --git a/extra/crds.yaml b/extra/crds.yaml index 8f008297..85fbcccb 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -939,7 +939,7 @@ spec: - host type: object type: object - celeryResultBackend: + celeryResultsBackend: description: |- Connection information for the celery backend database. Only works if `workers` (and `beat`) roles are set. diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index b0592665..e78c9097 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -13,7 +13,7 @@ use crate::{ }, }, resources::{ - celery_broker_connection_details, celery_result_backend_connection_details, + celery_broker_connection_details, celery_results_backend_connection_details, metadata_database_connection_details, }, v1alpha1::SupersetCluster, @@ -87,8 +87,8 @@ pub fn add_superset_config( pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: &SupersetCluster) { let ( Some(missing_result_backend_connection_details), - Some(celery_result_backend_connection_details), - ) = celery_result_backend_connection_details(superset) + Some(celery_results_backend_connection_details), + ) = celery_results_backend_connection_details(superset) else { return; }; @@ -97,15 +97,15 @@ pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: & return; }; - let result_backend_username_env = celery_result_backend_connection_details + let result_backend_username_env = celery_results_backend_connection_details .username_env .map(|env| env.name) .unwrap_or("".to_string()); - let result_backend_password_env = celery_result_backend_connection_details + let result_backend_password_env = celery_results_backend_connection_details .password_env .map(|env| env.name) .unwrap_or("".to_string()); - let result_backend_url_template = celery_result_backend_connection_details.url_template; + let result_backend_url_template = celery_results_backend_connection_details.url_template; let result_backend_host = missing_result_backend_connection_details.host; let result_backend_port = missing_result_backend_connection_details.port; let broker_url_template = celery_broker_connection_details.url_template; diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs index 7104226f..74077253 100644 --- a/rust/operator-binary/src/crd/databases.rs +++ b/rust/operator-binary/src/crd/databases.rs @@ -35,12 +35,12 @@ impl Deref for MetadataDatabaseConnection { #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub enum CeleryResultBackendConnection { +pub enum CeleryResultsBackendConnection { // Docs are on the struct Redis(RedisConnection), } -impl Deref for CeleryResultBackendConnection { +impl Deref for CeleryResultsBackendConnection { type Target = dyn CeleryDatabaseConnection; fn deref(&self) -> &Self::Target { @@ -50,10 +50,10 @@ impl Deref for CeleryResultBackendConnection { } } -impl CeleryResultBackendConnection { +impl CeleryResultsBackendConnection { pub fn as_python_parameters(&self) -> CeleryResultsBackendConnectionDetails { match &self { - CeleryResultBackendConnection::Redis(redis_connection) => { + CeleryResultsBackendConnection::Redis(redis_connection) => { CeleryResultsBackendConnectionDetails { host: stackable_operator::commons::networking::HostName::from( redis_connection.host.clone(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 63a56912..4f525231 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -37,7 +37,7 @@ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ crd::{ databases::{ - CeleryBrokerConnection, CeleryResultBackendConnection, MetadataDatabaseConnection, + CeleryBrokerConnection, CeleryResultsBackendConnection, MetadataDatabaseConnection, }, v1alpha1::SupersetRoleConfig, }, @@ -226,7 +226,7 @@ pub mod versioned { /// /// Ignored otherwise. #[serde(skip_serializing_if = "Option::is_none")] - pub celery_result_backend: Option, + pub celery_results_backend: Option, /// Connection information for the celery broker queue. /// diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index fbf2b943..ee5b94f4 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -153,15 +153,15 @@ pub fn build_worker_rolegroup_deployment( let metadata_database_connection_details = super::metadata_database_connection_details(superset); - let celery_result_backend_connection_details = - super::celery_result_backend_connection_details(superset); + let celery_results_backend_connection_details = + super::celery_results_backend_connection_details(superset); let celery_broker_connection_details = super::celery_broker_connection_details(superset); metadata_database_connection_details.add_to_container(&mut superset_cb); - if let (_, Some(celery_result_backend_connection_details)) = - &celery_result_backend_connection_details + if let (_, Some(celery_results_backend_connection_details)) = + &celery_results_backend_connection_details { - celery_result_backend_connection_details.add_to_container(&mut superset_cb); + celery_results_backend_connection_details.add_to_container(&mut superset_cb); } if let Some(celery_broker_connection_details) = celery_broker_connection_details { celery_broker_connection_details.add_to_container(&mut superset_cb); @@ -410,15 +410,15 @@ pub fn build_beat_rolegroup_deployment( let metadata_database_connection_details = super::metadata_database_connection_details(superset); - let celery_result_backend_connection_details = - super::celery_result_backend_connection_details(superset); + let celery_results_backend_connection_details = + super::celery_results_backend_connection_details(superset); let celery_broker_connection_details = super::celery_broker_connection_details(superset); metadata_database_connection_details.add_to_container(&mut superset_cb); - if let (_, Some(celery_result_backend_connection_details)) = - &celery_result_backend_connection_details + if let (_, Some(celery_results_backend_connection_details)) = + &celery_results_backend_connection_details { - celery_result_backend_connection_details.add_to_container(&mut superset_cb); + celery_results_backend_connection_details.add_to_container(&mut superset_cb); } if let Some(celery_broker_connection_details) = celery_broker_connection_details { celery_broker_connection_details.add_to_container(&mut superset_cb); diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs index b9f16992..3e94d08e 100644 --- a/rust/operator-binary/src/resources/mod.rs +++ b/rust/operator-binary/src/resources/mod.rs @@ -120,7 +120,7 @@ pub(crate) fn metadata_database_connection_details( ) } -pub(crate) fn celery_result_backend_connection_details( +pub(crate) fn celery_results_backend_connection_details( superset: &SupersetCluster, ) -> ( Option, @@ -130,17 +130,17 @@ pub(crate) fn celery_result_backend_connection_details( superset .spec .cluster_config - .celery_result_backend + .celery_results_backend .as_ref() .map(|backend| backend.as_python_parameters()), superset .spec .cluster_config - .celery_result_backend + .celery_results_backend .as_ref() .map(|backend| { backend.celery_connection_details_with_templating( - "CELERY_RESULT_BACKEND", + "CELERY_RESULTS_BACKEND", &TemplatingMechanism::BashEnvSubstitution, ) }), diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index ec1e3182..1bc33865 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -195,15 +195,15 @@ pub fn build_server_rolegroup_statefulset( let metadata_database_connection_details = super::metadata_database_connection_details(superset); - let celery_result_backend_connection_details = - super::celery_result_backend_connection_details(superset); + let celery_results_backend_connection_details = + super::celery_results_backend_connection_details(superset); let celery_broker_connection_details = super::celery_broker_connection_details(superset); metadata_database_connection_details.add_to_container(&mut superset_cb); - if let (_, Some(celery_result_backend_connection_details)) = - &celery_result_backend_connection_details + if let (_, Some(celery_results_backend_connection_details)) = + &celery_results_backend_connection_details { - celery_result_backend_connection_details.add_to_container(&mut superset_cb); + celery_results_backend_connection_details.add_to_container(&mut superset_cb); } if let Some(celery_broker_connection_details) = celery_broker_connection_details { celery_broker_connection_details.add_to_container(&mut superset_cb); diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index 33d44aa8..9705cf8e 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -53,7 +53,7 @@ spec: host: superset-postgresql database: superset credentialsSecretName: superset-postgresql-credentials - celeryResultBackend: + celeryResultsBackend: redis: host: superset-redis-master credentialsSecretName: superset-redis-credentials From 9fe679453744ff75bb1ce1f1bfe9874f7c6d91ba Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 26 Apr 2026 11:04:58 +0200 Subject: [PATCH 27/42] clippy: fix host lint. --- rust/operator-binary/src/crd/databases.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs index 74077253..855dee77 100644 --- a/rust/operator-binary/src/crd/databases.rs +++ b/rust/operator-binary/src/crd/databases.rs @@ -55,9 +55,7 @@ impl CeleryResultsBackendConnection { match &self { CeleryResultsBackendConnection::Redis(redis_connection) => { CeleryResultsBackendConnectionDetails { - host: stackable_operator::commons::networking::HostName::from( - redis_connection.host.clone(), - ), + host: redis_connection.host.clone(), port: redis_connection.port, database_id: redis_connection.database_id, } From 00d603b30c974a9fdb23fadd62ea02e272e68b26 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 27 Apr 2026 11:25:44 +0200 Subject: [PATCH 28/42] fix: enable proper logging for worker and beat. --- .../src/config/product_logging.rs | 62 ++++++++++++------- rust/operator-binary/src/config/superset.rs | 5 +- .../src/resources/configmap.rs | 2 +- .../src/resources/deployment.rs | 4 +- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/rust/operator-binary/src/config/product_logging.rs b/rust/operator-binary/src/config/product_logging.rs index 3d7da1ac..2995e56b 100644 --- a/rust/operator-binary/src/config/product_logging.rs +++ b/rust/operator-binary/src/config/product_logging.rs @@ -80,32 +80,52 @@ fn create_superset_config(log_config: &AutomaticContainerLogConfig, log_dir: &st import os from superset.utils.logging_configurator import LoggingConfigurator from pythonjsonlogger import jsonlogger + from celery.signals import setup_logging os.makedirs('{log_dir}', exist_ok=True) + _LOGGING_CONFIGURED = False + + + def _configure_root_logger(): + global _LOGGING_CONFIGURED + if _LOGGING_CONFIGURED: + return + _LOGGING_CONFIGURED = True + + logFormat = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' + + plainTextFormatter = logging.Formatter(logFormat) + jsonFormatter = jsonlogger.JsonFormatter(logFormat) + + consoleHandler = logging.StreamHandler() + consoleHandler.setLevel({console_log_level}) + consoleHandler.setFormatter(plainTextFormatter) + + fileHandler = logging.handlers.RotatingFileHandler( + '{log_dir}/{LOG_FILE}', + maxBytes=1048576, + backupCount=1, + ) + fileHandler.setLevel({file_log_level}) + fileHandler.setFormatter(jsonFormatter) + + rootLogger = logging.getLogger() + # Clear any handlers Celery/Flask/etc. already attached + rootLogger.handlers.clear() + rootLogger.setLevel({root_log_level}) + rootLogger.addHandler(consoleHandler) + rootLogger.addHandler(fileHandler) + + + @setup_logging.connect + def configure_celery_logging(**kwargs): + _configure_root_logger() + + class StackableLoggingConfigurator(LoggingConfigurator): def configure_logging(self, app_config: flask.config.Config, debug_mode: bool): - logFormat = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' - - plainTextFormatter = logging.Formatter(logFormat) - jsonFormatter = jsonlogger.JsonFormatter(logFormat) - - consoleHandler = logging.StreamHandler() - consoleHandler.setLevel({console_log_level}) - consoleHandler.setFormatter(plainTextFormatter) - - fileHandler = logging.handlers.RotatingFileHandler( - '{log_dir}/{LOG_FILE}', - maxBytes=1048576, - backupCount=1, - ) - fileHandler.setLevel({file_log_level}) - fileHandler.setFormatter(jsonFormatter) - - rootLogger = logging.getLogger() - rootLogger.setLevel({root_log_level}) - rootLogger.addHandler(consoleHandler) - rootLogger.addHandler(fileHandler) + _configure_root_logger() {loggers_config} ", diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index e78c9097..1c31a5cc 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -84,7 +84,10 @@ pub fn add_superset_config( Ok(()) } -pub(crate) fn append_celery_worker_config(config_file: &mut Vec, superset: &SupersetCluster) { +pub(crate) fn append_celery_connection_config( + config_file: &mut Vec, + superset: &SupersetCluster, +) { let ( Some(missing_result_backend_connection_details), Some(celery_results_backend_connection_details), diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs index 7c628344..f5c93eeb 100644 --- a/rust/operator-binary/src/resources/configmap.rs +++ b/rust/operator-binary/src/resources/configmap.rs @@ -125,7 +125,7 @@ pub fn build_rolegroup_config_map( })?; // We have to add a python class (no key) and cannot use the superset::config machinery. - config::superset::append_celery_worker_config(&mut config_file, superset); + config::superset::append_celery_connection_config(&mut config_file, superset); if let Some(footer) = temp_file_footer { writeln!(config_file, "{footer}").context(WriteToConfigFileStringSnafu)?; diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index ee5b94f4..24f272af 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -232,7 +232,7 @@ pub fn build_worker_rolegroup_deployment( prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - celery --app=superset.tasks.celery_app:app worker --loglevel=INFO --task-events & + celery --app=superset.tasks.celery_app:app worker --task-events & wait_for_termination $! {create_vector_shutdown_file_command} @@ -489,7 +489,7 @@ pub fn build_beat_rolegroup_deployment( prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid --loglevel=INFO & + celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid & wait_for_termination $! {create_vector_shutdown_file_command} From ed93462e3c0faa720361a7617de90c991dac3227 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 27 Apr 2026 11:31:38 +0200 Subject: [PATCH 29/42] docs: adapt changelog. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25087ac3..596c9b36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +### Added + +- Support for Async Queries via Celery. Two new roles, `worker` and `beat` ([#724]). +- Support for redis broker and results backend for celery workers in the CRD ([#724]). + ### Changed - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#717]). @@ -21,6 +26,7 @@ [#719]: https://github.com/stackabletech/superset-operator/pull/719 [#721]: https://github.com/stackabletech/superset-operator/pull/721 [#722]: https://github.com/stackabletech/superset-operator/pull/722 +[#724]: https://github.com/stackabletech/superset-operator/pull/724 ## [26.3.0] - 2026-03-16 From 6660043d532421393777e786b6936936ddb68ff7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 11:16:51 +0200 Subject: [PATCH 30/42] fix: use database id for results backend. --- rust/operator-binary/src/config/superset.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index 1c31a5cc..68ab7219 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -111,13 +111,14 @@ pub(crate) fn append_celery_connection_config( let result_backend_url_template = celery_results_backend_connection_details.url_template; let result_backend_host = missing_result_backend_connection_details.host; let result_backend_port = missing_result_backend_connection_details.port; + let result_backend_db = missing_result_backend_connection_details.database_id; let broker_url_template = celery_broker_connection_details.url_template; let celery_config = formatdoc!( r#" # CELERY ASYNC from flask_caching.backends.rediscache import RedisCache - RESULTS_BACKEND = RedisCache(host='{result_backend_host}', port={result_backend_port}, key_prefix='superset_results', username=os.path.expandvars('${{{result_backend_username_env}}}'), password=os.path.expandvars('${{{result_backend_password_env}}}')) + RESULTS_BACKEND = RedisCache(host='{result_backend_host}', port={result_backend_port}, db={result_backend_db}, key_prefix='superset_results', username=os.path.expandvars('${{{result_backend_username_env}}}'), password=os.path.expandvars('${{{result_backend_password_env}}}')) class CeleryConfig(object): broker_url = os.path.expandvars('{broker_url_template}') imports = ( From f6bcf048dc6e8376282e4668674b11fdf6f7205d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 11:34:29 +0200 Subject: [PATCH 31/42] fix: error variants in statefulset & deployment --- .../src/resources/deployment.rs | 28 ++++++++++++----- .../src/resources/statefulset.rs | 18 +++++++---- .../src/superset_controller.rs | 30 ++++++++++++++----- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 24f272af..387a367f 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -46,11 +46,11 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object defines no node role"))] - NoNodeRole, + #[snafu(display("object defines no '{role}' role"))] + MissingRole { role: String }, - #[snafu(display("object defines no node role-group"))] - NoNodeRoleGroup, + #[snafu(display("object defines no '{role}' rolegroup"))] + MissingRoleGroup { role: String }, #[snafu(display("invalid container name"))] InvalidContainerName { @@ -116,11 +116,17 @@ pub fn build_worker_rolegroup_deployment( sa_name: &str, merged_config: &SupersetConfig, ) -> Result { - let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; + let role = superset + .get_role(superset_role) + .with_context(|| MissingRoleSnafu { + role: superset_role.to_string(), + })?; let role_group = role .role_groups .get(&rolegroup_ref.role_group) - .context(NoNodeRoleGroupSnafu)?; + .with_context(|| MissingRoleGroupSnafu { + role: superset_role.to_string(), + })?; let recommended_object_labels = build_recommended_labels( superset, @@ -373,11 +379,17 @@ pub fn build_beat_rolegroup_deployment( sa_name: &str, merged_config: &SupersetConfig, ) -> Result { - let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; + let role = superset + .get_role(superset_role) + .with_context(|| MissingRoleSnafu { + role: superset_role.to_string(), + })?; let role_group = role .role_groups .get(&rolegroup_ref.role_group) - .context(NoNodeRoleGroupSnafu)?; + .with_context(|| MissingRoleGroupSnafu { + role: superset_role.to_string(), + })?; let recommended_object_labels = build_recommended_labels( superset, diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index 1bc33865..8e54c575 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -58,11 +58,11 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object defines no node role"))] - NoNodeRole, + #[snafu(display("object defines no '{role}' role"))] + MissingRole { role: String }, - #[snafu(display("object defines no node role-group"))] - NoNodeRoleGroup, + #[snafu(display("object defines no '{role}' rolegroup"))] + MissingRoleGroup { role: String }, #[snafu(display("invalid container name"))] InvalidContainerName { @@ -148,11 +148,17 @@ pub fn build_server_rolegroup_statefulset( sa_name: &str, merged_config: &SupersetConfig, ) -> Result { - let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; + let role = superset + .get_role(superset_role) + .with_context(|| MissingRoleSnafu { + role: superset_role.to_string(), + })?; let role_group = role .role_groups .get(&rolegroup_ref.role_group) - .context(NoNodeRoleGroupSnafu)?; + .with_context(|| MissingRoleGroupSnafu { + role: superset_role.to_string(), + })?; let recommended_object_labels = build_recommended_labels( superset, diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index b82d0af5..02aadd8a 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -60,8 +60,8 @@ pub struct Ctx { #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("object defines no node role"))] - NoNodeRole, + #[snafu(display("object defines no '{role}' role"))] + MissingRole { role: String }, #[snafu(display("failed to parse role: {source}"))] ParseRole { source: strum::ParseError }, @@ -249,7 +249,13 @@ pub async fn reconcile_superset( PropertyNameKind::Env, PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), ], - superset.spec.nodes.clone().context(NoNodeRoleSnafu)?, + superset + .spec + .nodes + .clone() + .with_context(|| MissingRoleSnafu { + role: SupersetRole::Node.to_string(), + })?, ), ), ( @@ -259,8 +265,13 @@ pub async fn reconcile_superset( PropertyNameKind::Env, PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), ], - // TODO: improve error - superset.spec.workers.clone().context(NoNodeRoleSnafu)?, + superset + .spec + .workers + .clone() + .with_context(|| MissingRoleSnafu { + role: SupersetRole::Worker.to_string(), + })?, ), ), ( @@ -270,8 +281,13 @@ pub async fn reconcile_superset( PropertyNameKind::Env, PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), ], - // TODO: improve error - superset.spec.beat.clone().context(NoNodeRoleSnafu)?, + superset + .spec + .beat + .clone() + .with_context(|| MissingRoleSnafu { + role: SupersetRole::Beat.to_string(), + })?, ), ), ] From 6ec305444b021f82e975fcb3ba836c45ed4ff875 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 11:55:19 +0200 Subject: [PATCH 32/42] fix: optional role definition in validated config. --- .../src/superset_controller.rs | 86 ++++++------------- 1 file changed, 26 insertions(+), 60 deletions(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 02aadd8a..f001f493 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -1,9 +1,9 @@ //! Ensures that `Pod`s are configured and running for each [`SupersetCluster`] -use std::{str::FromStr, sync::Arc}; +use std::{collections::HashMap, str::FromStr, sync::Arc}; use const_format::concatcp; use product_config::{ProductConfigManager, types::PropertyNameKind}; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ @@ -25,7 +25,7 @@ use stackable_operator::{ operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, }; -use strum::{EnumDiscriminants, IntoStaticStr}; +use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; use crate::{ OPERATOR_NAME, @@ -237,63 +237,29 @@ pub async fn reconcile_superset( .await .context(InvalidAuthenticationConfigSnafu)?; - let validated_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &transform_all_roles_to_config( - superset, - &[ - ( - SupersetRole::Node.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), - ], - superset - .spec - .nodes - .clone() - .with_context(|| MissingRoleSnafu { - role: SupersetRole::Node.to_string(), - })?, - ), - ), - ( - SupersetRole::Worker.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), - ], - superset - .spec - .workers - .clone() - .with_context(|| MissingRoleSnafu { - role: SupersetRole::Worker.to_string(), - })?, - ), - ), + let mut roles = HashMap::new(); + + // if the kubernetes executor is specified there will be no worker role as the pods + // are provisioned by airflow as defined by the task (default: one pod per task) + for role in SupersetRole::iter() { + if let Some(resolved_role) = superset.get_role(&role) { + roles.insert( + role.to_string(), ( - SupersetRole::Beat.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), - ], - superset - .spec - .beat - .clone() - .with_context(|| MissingRoleSnafu { - role: SupersetRole::Beat.to_string(), - })?, - ), + vec![ + PropertyNameKind::Env, + PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), + ], + resolved_role.clone(), ), - ] - .into(), - ) - .context(GenerateProductConfigSnafu)?, + ); + } + } + + let role_config = transform_all_roles_to_config(superset, &roles); + let validated_role_config = validate_all_roles_and_groups_config( + &resolved_product_image.product_version, + &role_config.context(GenerateProductConfigSnafu)?, &ctx.product_config, false, false, @@ -350,8 +316,8 @@ pub async fn reconcile_superset( let mut statefulset_cond_builder = StatefulSetConditionBuilder::default(); let mut deployment_cond_builder = DeploymentConditionBuilder::default(); - for (superset_role_str, role_config) in validated_config { - let superset_role = SupersetRole::from_str(&superset_role_str).context(ParseRoleSnafu)?; + for (role_name, role_config) in validated_role_config.iter() { + let superset_role = SupersetRole::from_str(role_name).context(ParseRoleSnafu)?; for (rolegroup_name, rolegroup_config) in role_config.iter() { let rolegroup = superset.rolegroup_ref(&superset_role, rolegroup_name); From f9b0f86a0cae7637e3f17242acf5bfe03dedd8a5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 11:59:26 +0200 Subject: [PATCH 33/42] docs: improve beat replica documentation --- .../pages/usage-guide/celery-async-queries.adoc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc index e6d1775e..c64fce90 100644 --- a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc +++ b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc @@ -4,7 +4,10 @@ In order to support long running queries that exceed the web requests timeout, an asynchronous backend has to be configured. This requires two more roles to be configured besides the existing `nodes`. The (celery) `workers` and a `beat` for scheduling. -The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored and set to `1`. +The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored and set to `1`. +This avoids multiple beat instances scheduling the same tasks at the same time. + +If no beat or scheduling is required, async queries keep work without a beat instance. The following example shows additional required settings to enable async queries: @@ -22,11 +25,15 @@ spec: host: superset-redis-master port: 6379 credentialsSecretName: superset-redis-results-backend-credentials + nodes: + roleGroups: + default: + replicas: 1 workers: roleGroups: default: replicas: 2 - beat: + beat: # optional roleGroups: default: # Only 1 or 0 instances possible. From 08a377db1faea7553ea1845e7c95f829406dac97 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 12:11:43 +0200 Subject: [PATCH 34/42] docs: fix linter --- .../superset/pages/usage-guide/celery-async-queries.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc index c64fce90..5530d9be 100644 --- a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc +++ b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc @@ -4,7 +4,7 @@ In order to support long running queries that exceed the web requests timeout, an asynchronous backend has to be configured. This requires two more roles to be configured besides the existing `nodes`. The (celery) `workers` and a `beat` for scheduling. -The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored and set to `1`. +The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored and set to `1`. This avoids multiple beat instances scheduling the same tasks at the same time. If no beat or scheduling is required, async queries keep work without a beat instance. From fc62924e753750ddd3077e026684b47e7048450a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 12:21:43 +0200 Subject: [PATCH 35/42] fix: remove obsolete logging TODOs. --- rust/operator-binary/src/resources/deployment.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 387a367f..37ba3c7c 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -225,8 +225,6 @@ pub fn build_worker_rolegroup_deployment( "pipefail".to_string(), "-c".to_string(), ]) - // TODO: Without --loglevel=INFO, the worker does not log anyhing. - // This should be investigated and configurable. .args(vec![formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} @@ -488,8 +486,6 @@ pub fn build_beat_rolegroup_deployment( "pipefail".to_string(), "-c".to_string(), ]) - // TODO: Without --loglevel=INFO, the beat does not log anyhing. - // This should be investigated and configurable. .args(vec![formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} From e9f8b45d11fd500eb02bb791251f448126b6065e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 12:53:30 +0200 Subject: [PATCH 36/42] fix: remove c&p code comment --- rust/operator-binary/src/superset_controller.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index f001f493..82219f45 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -239,8 +239,6 @@ pub async fn reconcile_superset( let mut roles = HashMap::new(); - // if the kubernetes executor is specified there will be no worker role as the pods - // are provisioned by airflow as defined by the task (default: one pod per task) for role in SupersetRole::iter() { if let Some(resolved_role) = superset.get_role(&role) { roles.insert( From b2f9e60ebbb96e448f064abfb6d3ce5f8974576f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 28 Apr 2026 13:05:39 +0200 Subject: [PATCH 37/42] fix: remove "missing" from celery results backend connection details. --- rust/operator-binary/src/config/superset.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/config/superset.rs b/rust/operator-binary/src/config/superset.rs index 68ab7219..c1097813 100644 --- a/rust/operator-binary/src/config/superset.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -89,7 +89,7 @@ pub(crate) fn append_celery_connection_config( superset: &SupersetCluster, ) { let ( - Some(missing_result_backend_connection_details), + Some(additional_celery_results_backend_connection_details), Some(celery_results_backend_connection_details), ) = celery_results_backend_connection_details(superset) else { @@ -109,9 +109,9 @@ pub(crate) fn append_celery_connection_config( .map(|env| env.name) .unwrap_or("".to_string()); let result_backend_url_template = celery_results_backend_connection_details.url_template; - let result_backend_host = missing_result_backend_connection_details.host; - let result_backend_port = missing_result_backend_connection_details.port; - let result_backend_db = missing_result_backend_connection_details.database_id; + let result_backend_host = additional_celery_results_backend_connection_details.host; + let result_backend_port = additional_celery_results_backend_connection_details.port; + let result_backend_db = additional_celery_results_backend_connection_details.database_id; let broker_url_template = celery_broker_connection_details.url_template; let celery_config = formatdoc!( From 234a4a39aa2086a73a623f7ae7c0b87770be4ff0 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 3 May 2026 11:43:23 +0200 Subject: [PATCH 38/42] fix: add Deployment to watch. --- rust/operator-binary/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 2e7f8639..700605a3 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -12,7 +12,7 @@ use stackable_operator::{ crd::authentication::core, eos::EndOfSupportChecker, k8s_openapi::api::{ - apps::v1::StatefulSet, + apps::v1::{Deployment, StatefulSet}, batch::v1::Job, core::v1::{ConfigMap, Service}, }, @@ -152,6 +152,11 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), ) + // Required for workers and beat. + .owns( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ) .watches( client.get_api::>(&()), watcher::Config::default(), From 5690038d7c796ce7a8df5caf2ce2bbee4dc4a542 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 4 May 2026 07:58:00 +0200 Subject: [PATCH 39/42] docs: consolidate statefulset method header. --- rust/operator-binary/src/resources/statefulset.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs index 8e54c575..2c49032c 100644 --- a/rust/operator-binary/src/resources/statefulset.rs +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -134,9 +134,6 @@ pub enum Error { type Result = std::result::Result; /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding -/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (via [`build_node_rolegroup_headless_service`] and metrics from [`build_node_rolegroup_metrics_service`]). #[allow(clippy::too_many_arguments)] pub fn build_server_rolegroup_statefulset( superset: &SupersetCluster, From d01bf14ca01109d4c60475064995c286179c9e8d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 4 May 2026 08:50:03 +0200 Subject: [PATCH 40/42] docs: remove double heading --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e530beea..da77548a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,6 @@ - Support for Async Queries via Celery. Two new roles, `worker` and `beat` ([#724]). - Support for redis broker and results backend for celery workers in the CRD ([#724]). - -### Added - - BREAKING: Add required CLI argument and env var to set the image repository used to construct final product image names: `IMAGE_REPOSITORY` (`--image-repository`), eg. `oci.example.org/my/namespace` ([#726]). ### Changed From 1d293a9347a6a7d6d7025802d6b421aa5e62d8fb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 4 May 2026 10:44:45 +0200 Subject: [PATCH 41/42] chore: add warning if beat replicas > 1. --- rust/operator-binary/src/resources/deployment.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs index 37ba3c7c..cd9bf33a 100644 --- a/rust/operator-binary/src/resources/deployment.rs +++ b/rust/operator-binary/src/resources/deployment.rs @@ -592,6 +592,15 @@ pub fn build_beat_rolegroup_deployment( pod_template.merge_from(role.config.pod_overrides.clone()); pod_template.merge_from(role_group.config.pod_overrides.clone()); + let replicas = if let Some(replicas) = role_group.replicas { + if replicas > 1 { + tracing::warn! {"replicas for role `beat` set to greater `1`. Multiple beat instances are not allowed. Setting to `1` replica."} + } + 1 + } else { + 0 + }; + Ok(Deployment { metadata: ObjectMetaBuilder::new() .name_and_namespace(superset) @@ -608,10 +617,7 @@ pub fn build_beat_rolegroup_deployment( spec: Some(DeploymentSpec { // Beat should always only be one Beat instance at a time. // We ignore values > 1, 0 is a possible value still. - replicas: role_group - .replicas - .map(i32::from) - .map(|r| if r >= 1 { 1 } else { 0 }), + replicas: Some(replicas), selector: LabelSelector { match_labels: Some( Labels::role_group_selector( From 93ef2c9a47628c8e5b367f9ef7d4f8a1e1a7eb38 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 5 May 2026 09:08:23 +0200 Subject: [PATCH 42/42] test: Use external-unstable for kuttl tests --- tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 index 9705cf8e..6cb43f76 100644 --- a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -65,6 +65,8 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: + roleConfig: + listenerClass: external-unstable config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}