diff --git a/CHANGELOG.md b/CHANGELOG.md index ed48a959..da77548a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Added +- Support for Async Queries via Celery. Two new roles, `worker` and `beat` ([#724]). +- Support for redis broker and results backend for celery workers in the CRD ([#724]). - BREAKING: Add required CLI argument and env var to set the image repository used to construct final product image names: `IMAGE_REPOSITORY` (`--image-repository`), eg. `oci.example.org/my/namespace` ([#726]). ### Changed @@ -25,6 +27,7 @@ [#719]: https://github.com/stackabletech/superset-operator/pull/719 [#721]: https://github.com/stackabletech/superset-operator/pull/721 [#722]: https://github.com/stackabletech/superset-operator/pull/722 +[#724]: https://github.com/stackabletech/superset-operator/pull/724 [#726]: https://github.com/stackabletech/superset-operator/pull/726 ## [26.3.0] - 2026-03-16 diff --git a/deploy/helm/chart_testing.yaml b/deploy/helm/chart_testing.yaml index 82b39c26..253af46d 100644 --- a/deploy/helm/chart_testing.yaml +++ b/deploy/helm/chart_testing.yaml @@ -1,3 +1,4 @@ +--- remote: origin target-branch: main chart-dirs: diff --git a/deploy/helm/superset-operator/templates/clusterrole-operator.yaml b/deploy/helm/superset-operator/templates/clusterrole-operator.yaml index 567dbabd..185fdc3b 100644 --- a/deploy/helm/superset-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/superset-operator/templates/clusterrole-operator.yaml @@ -64,6 +64,18 @@ rules: resourceNames: - {{ include "operator.name" . }}-clusterrole # StatefulSet created per role group. Applied via SSA and tracked for orphan cleanup. + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - create + - delete + - list + - patch + - update + - watch - apiGroups: - apps resources: diff --git a/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc new file mode 100644 index 00000000..5530d9be --- /dev/null +++ b/docs/modules/superset/pages/usage-guide/celery-async-queries.adoc @@ -0,0 +1,43 @@ += Async Queries via Celery +:description: Schedule Apache Superset SQL queries on Celery workers. + +In order to support long running queries that exceed the web requests timeout, an asynchronous backend has to be configured. +This requires two more roles to be configured besides the existing `nodes`. The (celery) `workers` and a `beat` for scheduling. + +The beat role is limited to `1` or `0` replicas only. Replicas greater than one are ignored and set to `1`. +This avoids multiple beat instances scheduling the same tasks at the same time. + +If no beat or scheduling is required, async queries keep work without a beat instance. + +The following example shows additional required settings to enable async queries: + +[source,yaml] +---- +spec: + clusterConfig: + celeryBroker: + redis: + host: superset-redis-master + port: 6379 + credentialsSecretName: superset-redis-broker-credentials + celeryResultsBackend: + redis: + host: superset-redis-master + port: 6379 + credentialsSecretName: superset-redis-results-backend-credentials + nodes: + roleGroups: + default: + replicas: 1 + workers: + roleGroups: + default: + replicas: 2 + beat: # optional + roleGroups: + default: + # Only 1 or 0 instances possible. + replicas: 1 +---- + +This is not a complete example. The main `nodes` role for the webserver and e.g. the metadata database are missing. diff --git a/docs/modules/superset/pages/usage-guide/database-connections.adoc b/docs/modules/superset/pages/usage-guide/database-connections.adoc index 355762fa..06c12827 100644 --- a/docs/modules/superset/pages/usage-guide/database-connections.adoc +++ b/docs/modules/superset/pages/usage-guide/database-connections.adoc @@ -3,8 +3,11 @@ Superset requires a metadata database for storing slices, connections, tables, dashboards and other metadata. The actual connection string is calculated by the operator so that the user does not need to remember the exact structure. +Using async queries with celery workers also requires a broker and a results backend. -== Typed connections +== Metadata database + +=== PostgreSQL [source,yaml] ---- @@ -19,7 +22,7 @@ spec: <1> A reference to one of the supported database backends (e.g. `postgresql`). <2> A reference to a Secret which must contain the two keys `username` and `password`. -== Generic connections +=== Generic connections Alternatively, these connections can also be defined in full in a referenced Secret: @@ -33,3 +36,75 @@ spec: ---- <1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `postgresql://superset:superset@superset-postgresql/superset` + +== Broker + +The broker queue is required to schedule or pass queries on the celery workers. + +=== Redis + +A redis broker can be configured in the `clusterConfig`: + +[source,yaml] +---- +spec: + clusterConfig: + celeryBroker: + redis: + host: redis-master + port: 6379 + credentialsSecretName: superset-redis-credentials +---- + +=== Generic connections + +Alternatively, these connections can also be defined in full in a referenced Secret: + +[source,yaml] +---- +spec: + clusterConfig: + celeryBroker: + generic: + connectionUrlSecretName: superset-redis-broker-url # <1> +---- + +<1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `redis://:redis@redis-master/0` + +== Results Backend + +Celery workers store their results in the configured results backend. + +=== Redis + +A redis broker can be configured in the `clusterConfig`: + +[source,yaml] +---- +spec: + clusterConfig: + celeryResultsBackend: + redis: + host: redis-master + port: 6379 + credentialsSecretName: superset-redis-credentials +---- + +=== S3 + +Currently not supported. + +=== Generic connections + +Alternatively, these connections can also be defined in full in a referenced Secret: + +[source,yaml] +---- +spec: + clusterConfig: + celeryResultsBackend: + generic: + connectionUrlSecretName: superset-redis-results-backend-url # <1> +---- + +<1> A reference to a Secret which must contain the single key `connectionUrl` e.g. `redis://:redis@redis-master/0` diff --git a/extra/crds.yaml b/extra/crds.yaml index 5678350b..5add8d90 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -26,300 +26,1885 @@ spec: Find more information on how to use it and the resources that the operator generates in the [operator documentation](https://docs.stackable.tech/home/nightly/superset/). properties: - clusterConfig: + beat: description: |- - Settings that affect all roles and role groups. - The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. + This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing + all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable + at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured + on role level. There is also a second form of config, which can only be configured + at role level, the `roleConfig`. + You can learn more about this in the + [Roles and role group concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups). + nullable: true properties: - authentication: - default: [] - description: List of AuthenticationClasses used to authenticate users. - items: - properties: - authenticationClass: - description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users - type: string - oidc: - description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. - nullable: true - properties: - clientAuthenticationMethod: - default: client_secret_basic - description: 'The client authentication method used when communicating with the token endpoint. Defaults to `client_secret_basic`. The required contents of `clientCredentialsSecret` depend on the chosen method: secret-based methods (`client_secret_basic`, `client_secret_post`, `client_secret_jwt`) expect a client secret, while `private_key_jwt` expects a private key.' - enum: - - client_secret_basic - - client_secret_post - - client_secret_jwt - - private_key_jwt - - none - type: string - clientCredentialsSecret: - description: |- - A reference to the OIDC client credentials secret. The secret contains - the client id and secret. - type: string - extraScopes: - default: [] - description: An optional list of extra scopes which get merged with the scopes defined in the AuthenticationClass - items: - type: string - type: array - required: - - clientCredentialsSecret - type: object - syncRolesAt: - default: Registration - description: |- - If we should replace ALL the user's roles each login, or only on registration. - Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` - enum: - - Registration - - Login - type: string - userRegistration: - default: true - description: |- - Allow users who are not already in the FAB DB. - Gets mapped to `AUTH_USER_REGISTRATION` - type: boolean - userRegistrationRole: - default: Public - description: |- - This role will be given in addition to any AUTH_ROLES_MAPPING. - Gets mapped to `AUTH_USER_REGISTRATION_ROLE` - type: string - required: - - authenticationClass - type: object - type: array - authorization: - description: |- - Authorization options for Superset. - - Currently only role assignment is supported. This means that roles are assigned to users in - OPA but, due to the way Superset is implemented, the database also needs to be updated - to reflect these assignments. - Therefore, user roles and permissions must already exist in the Superset database before - they can be assigned to a user. - Warning: Any user roles assigned with the Superset UI are discarded. - nullable: true + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} properties: - roleMappingFromOpa: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null description: |- - Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) - and the name of the Rego package containing your authorization rules. - Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) - to learn how to deploy Rego authorization rules with OPA. + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: - cache: - default: - entryTimeToLive: 30s - maxEntries: 10000 - description: Configuration for an Superset internal cache for calls to OPA + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. properties: - entryTimeToLive: - default: 30s - description: Time to live per entry + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + type: string + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + clusterConfig: + description: |- + Settings that affect all roles and role groups. + The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. + properties: + authentication: + default: [] + description: List of AuthenticationClasses used to authenticate users. + items: + properties: + authenticationClass: + description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users + type: string + oidc: + description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. + nullable: true + properties: + clientAuthenticationMethod: + default: client_secret_basic + description: 'The client authentication method used when communicating with the token endpoint. Defaults to `client_secret_basic`. The required contents of `clientCredentialsSecret` depend on the chosen method: secret-based methods (`client_secret_basic`, `client_secret_post`, `client_secret_jwt`) expect a client secret, while `private_key_jwt` expects a private key.' + enum: + - client_secret_basic + - client_secret_post + - client_secret_jwt + - private_key_jwt + - none + type: string + clientCredentialsSecret: + description: |- + A reference to the OIDC client credentials secret. The secret contains + the client id and secret. + type: string + extraScopes: + default: [] + description: An optional list of extra scopes which get merged with the scopes defined in the AuthenticationClass + items: + type: string + type: array + required: + - clientCredentialsSecret + type: object + syncRolesAt: + default: Registration + description: |- + If we should replace ALL the user's roles each login, or only on registration. + Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` + enum: + - Registration + - Login + type: string + userRegistration: + default: true + description: |- + Allow users who are not already in the FAB DB. + Gets mapped to `AUTH_USER_REGISTRATION` + type: boolean + userRegistrationRole: + default: Public + description: |- + This role will be given in addition to any AUTH_ROLES_MAPPING. + Gets mapped to `AUTH_USER_REGISTRATION_ROLE` + type: string + required: + - authenticationClass + type: object + type: array + authorization: + description: |- + Authorization options for Superset. + + Currently only role assignment is supported. This means that roles are assigned to users in + OPA but, due to the way Superset is implemented, the database also needs to be updated + to reflect these assignments. + Therefore, user roles and permissions must already exist in the Superset database before + they can be assigned to a user. + Warning: Any user roles assigned with the Superset UI are discarded. + nullable: true + properties: + roleMappingFromOpa: + description: |- + Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + and the name of the Rego package containing your authorization rules. + Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) + to learn how to deploy Rego authorization rules with OPA. + properties: + cache: + default: + entryTimeToLive: 30s + maxEntries: 10000 + description: Configuration for an Superset internal cache for calls to OPA + properties: + entryTimeToLive: + default: 30s + description: Time to live per entry + type: string + maxEntries: + default: 10000 + description: |- + Maximum number of entries in the cache; If this threshold is reached then the least + recently used item is removed. + format: uint32 + minimum: 0.0 + type: integer + type: object + configMapName: + description: |- + The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + for the OPA stacklet that should be used for authorization requests. + type: string + package: + description: The name of the Rego package containing the Rego rules for the product. + nullable: true + type: string + required: + - configMapName + type: object + required: + - roleMappingFromOpa + type: object + celeryBroker: + description: |- + Connection information for the celery broker queue. + + Only works if `workers` (and `beat`) roles are set. + Ignored otherwise. + nullable: true + oneOf: + - required: + - redis + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + redis: + description: |- + Connection settings for a [Redis](https://redis.io/) instance. + + Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the Redis server. + type: string + databaseId: + default: 0 + description: |- + Numeric index of the Redis logical database to use. Defaults to `0`. + + Redis supports multiple logical databases within a single instance, identified by an + integer index. Database `0` is the default. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + host: + description: Hostname or IP address of the Redis server. + type: string + port: + default: 6379 + description: Port the Redis server is listening on. Defaults to `6379`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - host + type: object + type: object + celeryResultsBackend: + description: |- + Connection information for the celery backend database. + Only works if `workers` (and `beat`) roles are set. + + Ignored otherwise. + nullable: true + oneOf: + - required: + - redis + properties: + redis: + description: |- + Connection settings for a [Redis](https://redis.io/) instance. + + Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the Redis server. + type: string + databaseId: + default: 0 + description: |- + Numeric index of the Redis logical database to use. Defaults to `0`. + + Redis supports multiple logical databases within a single instance, identified by an + integer index. Database `0` is the default. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + host: + description: Hostname or IP address of the Redis server. + type: string + port: + default: 6379 + description: Port the Redis server is listening on. Defaults to `6379`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - host + type: object + type: object + clusterOperation: + default: + reconciliationPaused: false + stopped: false + description: Cluster operations like pause reconciliation or cluster stop. + properties: + reconciliationPaused: + default: false + description: |- + Flag to stop cluster reconciliation by the operator. This means that all changes in the + custom resource spec are ignored until this flag is set to false or removed. The operator + will however still watch the deployed resources at the time and update the custom resource + status field. + If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over + `stopped` and stop the reconciliation immediately. + type: boolean + stopped: + default: false + description: |- + Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, + ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 + and therefore stopped and removed. + If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation + and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. + type: boolean + type: object + credentialsSecretName: + description: |- + The name of the Secret object containing the admin user credentials. + Read the + [getting started guide first steps](https://docs.stackable.tech/home/nightly/superset/getting_started/first_steps) + to find out more. + type: string + mapboxSecret: + description: |- + The name of a Secret object. + The Secret should contain a key `connections.mapboxApiKey`. + This is the API key required for map charts to work that use mapbox. + The token should be in the JWT format. + nullable: true + type: string + metadataDatabase: + description: Configure the database where Superset stores all its internal metadata. + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + description: |- + A generic SQLAlchemy database connection for database types not covered by a dedicated variant. + + Use this when you need to connect to a SQLAlchemy-compatible database that does not have a + first-class connection type. The complete connection URL is read from a Secret, giving the user + full control over the connection string including any driver-specific options. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete SQLAlchemy URL. + type: string + required: + - connectionUrlSecretName + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + type: object + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + nullable: true + type: string + required: + - credentialsSecretName + - metadataDatabase + type: object + image: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string + type: object + nodes: + description: |- + This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing + all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable + at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured + on role level. There is also a second form of config, which can only be configured + at role level, the `roleConfig`. + You can learn more about this in the + [Roles and role group concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups). + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + type: string + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + superset: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: CPU and memory limits for Superset pods + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + rowLimit: + description: Row limit when requesting chart data. Corresponds to ROW_LIMIT. + format: int32 + nullable: true + type: integer + webserverTimeout: + description: |- + Maximum time period a Superset request can take before timing out. This + setting affects the maximum duration a query to an underlying datasource + can take. If you get timeout errors before your query returns the result + you may need to increase this timeout. Corresponds to + SUPERSET_WEBSERVER_TIMEOUT. + format: uint32 + minimum: 0.0 + nullable: true + type: integer + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + superset_config.py: + additionalProperties: type: string - maxEntries: - default: 10000 - description: |- - Maximum number of entries in the cache; If this threshold is reached then the least - recently used item is removed. - format: uint32 - minimum: 0.0 - type: integer - type: object - configMapName: - description: |- - The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) - for the OPA stacklet that should be used for authorization requests. - type: string - package: - description: The name of the Rego package containing the Rego rules for the product. - nullable: true - type: string - required: - - configMapName - type: object - required: - - roleMappingFromOpa - type: object - clusterOperation: - default: - reconciliationPaused: false - stopped: false - description: Cluster operations like pause reconciliation or cluster stop. - properties: - reconciliationPaused: - default: false - description: |- - Flag to stop cluster reconciliation by the operator. This means that all changes in the - custom resource spec are ignored until this flag is set to false or removed. The operator - will however still watch the deployed resources at the time and update the custom resource - status field. - If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over - `stopped` and stop the reconciliation immediately. - type: boolean - stopped: - default: false - description: |- - Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, - ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 - and therefore stopped and removed. - If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation - and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. - type: boolean - type: object - credentialsSecretName: - description: |- - The name of the Secret object containing the admin user credentials. - Read the - [getting started guide first steps](https://docs.stackable.tech/home/nightly/superset/getting_started/first_steps) - to find out more. - type: string - mapboxSecret: - description: |- - The name of a Secret object. - The Secret should contain a key `connections.mapboxApiKey`. - This is the API key required for map charts to work that use mapbox. - The token should be in the JWT format. - nullable: true - type: string - metadataDatabase: - description: Configure the database where Superset stores all its internal metadata. - oneOf: - - required: - - postgresql - - required: - - generic - properties: - generic: - description: |- - A generic SQLAlchemy database connection for database types not covered by a dedicated variant. + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. - Use this when you need to connect to a SQLAlchemy-compatible database that does not have a - first-class connection type. The complete connection URL is read from a Secret, giving the user - full control over the connection string including any driver-specific options. - properties: - connectionUrlSecretName: - description: The name of the Secret that contains an `connectionUrl` key with the complete SQLAlchemy URL. - type: string - required: - - connectionUrlSecretName - type: object - postgresql: - description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. - properties: - credentialsSecretName: - description: |- - Name of a Secret containing the `username` and `password` keys used to authenticate - against the PostgreSQL server. - type: string - database: - description: Name of the database (schema) to connect to. - type: string - host: - description: Hostname or IP address of the PostgreSQL server. + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: type: string - parameters: - additionalProperties: - type: string - default: {} - description: |- - Additional map of JDBC connection parameters to append to the connection URL. The given - `HashMap` will be converted to query parameters in the form of - `?param1=value1¶m2=value2`. - type: object - port: - default: 5432 - description: Port the PostgreSQL server is listening on. Defaults to `5432`. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecretName - - database - - host - type: object + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object type: object - vectorAggregatorConfigMapName: - description: |- - Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). - It must contain the key `ADDRESS` with the address of the Vector aggregator. - Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) - to learn how to configure log aggregation with Vector. - nullable: true - type: string required: - - credentialsSecretName - - metadataDatabase + - roleGroups type: object - image: - anyOf: - - required: - - custom - - productVersion - - required: - - productVersion + objectOverrides: + default: [] description: |- - Specify which image to use, the easiest way is to only configure the `productVersion`. - You can also configure a custom image registry to pull from, as well as completely custom - images. - - Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) - for details. - properties: - custom: - description: |- - Provide a custom container image. - - Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` - type: string - productVersion: - description: Version of the product, e.g. `1.4.1`. - type: string - pullPolicy: - default: Always - description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' - enum: - - IfNotPresent - - Always - - Never - type: string - pullSecrets: - description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' - items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. - properties: - name: - description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - name - type: object - nullable: true - type: array - repo: - description: |- - The repository on the container image registry where the container image is located, e.g. - `oci.example.com/namespace`. + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. - If not specified, the operator will use the image registry provided via the operator - environment options. - nullable: true - type: string - stackableVersion: - description: |- - Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. - If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly - operator or a PR version, it will use the nightly `0.0.0-dev` image. - nullable: true - type: string - type: object - nodes: + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + workers: description: |- This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable @@ -1048,20 +2633,6 @@ spec: required: - roleGroups type: object - objectOverrides: - default: [] - description: |- - A list of generic Kubernetes objects, which are merged into the objects that the operator - creates. - - List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. - - Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) - for more information. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array required: - clusterConfig - image diff --git a/rust/operator-binary/src/commands.rs b/rust/operator-binary/src/config/commands.rs similarity index 100% rename from rust/operator-binary/src/commands.rs rename to rust/operator-binary/src/config/commands.rs diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs new file mode 100644 index 00000000..c09ab723 --- /dev/null +++ b/rust/operator-binary/src/config/mod.rs @@ -0,0 +1,3 @@ +pub mod commands; +pub mod product_logging; +pub mod superset; diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/config/product_logging.rs similarity index 59% rename from rust/operator-binary/src/product_logging.rs rename to rust/operator-binary/src/config/product_logging.rs index c38a9038..2995e56b 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/config/product_logging.rs @@ -1,6 +1,6 @@ use std::fmt::{Display, Write}; -use snafu::Snafu; +use indoc::formatdoc; use stackable_operator::{ builder::configmap::ConfigMapBuilder, kube::Resource, @@ -15,24 +15,6 @@ use stackable_operator::{ use crate::crd::STACKABLE_LOG_DIR; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object has no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to retrieve the ConfigMap [{cm_name}]"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - #[snafu(display("failed to retrieve the entry [{entry}] for ConfigMap [{cm_name}]"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, - }, -} - -type Result = std::result::Result; - pub const LOG_CONFIG_FILE: &str = "log_config.py"; const LOG_FILE: &str = "superset.py.json"; @@ -44,8 +26,7 @@ pub fn extend_config_map_with_log_config( main_container: &C, vector_container: &C, cm_builder: &mut ConfigMapBuilder, -) -> Result<()> -where +) where C: Clone + Ord + Display, K: Resource, { @@ -75,8 +56,6 @@ where product_logging::framework::create_vector_config(rolegroup, vector_log_config), ); } - - Ok(()) } fn create_superset_config(log_config: &AutomaticContainerLogConfig, log_dir: &str) -> String { @@ -94,41 +73,62 @@ fn create_superset_config(log_config: &AutomaticContainerLogConfig, log_dir: &st ); }); - format!( - "\ -import flask.config -import logging -import os -from superset.utils.logging_configurator import LoggingConfigurator -from pythonjsonlogger import jsonlogger - -os.makedirs('{log_dir}', exist_ok=True) - -class StackableLoggingConfigurator(LoggingConfigurator): - def configure_logging(self, app_config: flask.config.Config, debug_mode: bool): - logFormat = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' - - plainTextFormatter = logging.Formatter(logFormat) - jsonFormatter = jsonlogger.JsonFormatter(logFormat) - - consoleHandler = logging.StreamHandler() - consoleHandler.setLevel({console_log_level}) - consoleHandler.setFormatter(plainTextFormatter) - - fileHandler = logging.handlers.RotatingFileHandler( - '{log_dir}/{LOG_FILE}', - maxBytes=1048576, - backupCount=1, - ) - fileHandler.setLevel({file_log_level}) - fileHandler.setFormatter(jsonFormatter) - - rootLogger = logging.getLogger() - rootLogger.setLevel({root_log_level}) - rootLogger.addHandler(consoleHandler) - rootLogger.addHandler(fileHandler) - -{loggers_config}", + formatdoc!( + " + import flask.config + import logging + import os + from superset.utils.logging_configurator import LoggingConfigurator + from pythonjsonlogger import jsonlogger + from celery.signals import setup_logging + + os.makedirs('{log_dir}', exist_ok=True) + + _LOGGING_CONFIGURED = False + + + def _configure_root_logger(): + global _LOGGING_CONFIGURED + if _LOGGING_CONFIGURED: + return + _LOGGING_CONFIGURED = True + + logFormat = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' + + plainTextFormatter = logging.Formatter(logFormat) + jsonFormatter = jsonlogger.JsonFormatter(logFormat) + + consoleHandler = logging.StreamHandler() + consoleHandler.setLevel({console_log_level}) + consoleHandler.setFormatter(plainTextFormatter) + + fileHandler = logging.handlers.RotatingFileHandler( + '{log_dir}/{LOG_FILE}', + maxBytes=1048576, + backupCount=1, + ) + fileHandler.setLevel({file_log_level}) + fileHandler.setFormatter(jsonFormatter) + + rootLogger = logging.getLogger() + # Clear any handlers Celery/Flask/etc. already attached + rootLogger.handlers.clear() + rootLogger.setLevel({root_log_level}) + rootLogger.addHandler(consoleHandler) + rootLogger.addHandler(fileHandler) + + + @setup_logging.connect + def configure_celery_logging(**kwargs): + _configure_root_logger() + + + class StackableLoggingConfigurator(LoggingConfigurator): + def configure_logging(self, app_config: flask.config.Config, debug_mode: bool): + _configure_root_logger() + + {loggers_config} + ", root_log_level = log_config.root_log_level().to_python_expression(), console_log_level = log_config .console diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config/superset.rs similarity index 81% rename from rust/operator-binary/src/config.rs rename to rust/operator-binary/src/config/superset.rs index b94dc44a..c1097813 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config/superset.rs @@ -1,15 +1,22 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, io::Write}; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_operator::crd::authentication::{ldap, oidc}; -use crate::crd::{ - SupersetConfigOptions, - authentication::{ - self, DEFAULT_OIDC_PROVIDER, SupersetAuthenticationClassResolved, - SupersetClientAuthenticationDetailsResolved, +use crate::{ + crd::{ + SupersetConfigOptions, + authentication::{ + self, DEFAULT_OIDC_PROVIDER, SupersetAuthenticationClassResolved, + SupersetClientAuthenticationDetailsResolved, + }, }, + resources::{ + celery_broker_connection_details, celery_results_backend_connection_details, + metadata_database_connection_details, + }, + v1alpha1::SupersetCluster, }; #[derive(Snafu, Debug)] @@ -39,15 +46,19 @@ pub const PYTHON_IMPORTS: &[&str] = &[ pub fn add_superset_config( config: &mut BTreeMap, + superset: &SupersetCluster, authentication_config: &SupersetClientAuthenticationDetailsResolved, ) -> Result<(), Error> { + let metadata_database_url_template = + metadata_database_connection_details(superset).url_template; + config.insert( SupersetConfigOptions::SecretKey.to_string(), "os.environ.get('SECRET_KEY')".to_owned(), ); config.insert( SupersetConfigOptions::SqlalchemyDatabaseUri.to_string(), - "os.path.expandvars(os.environ.get('SQLALCHEMY_DATABASE_URI'))".to_owned(), + format!("os.path.expandvars('{metadata_database_url_template}')"), ); config.insert( SupersetConfigOptions::StatsLogger.to_string(), @@ -70,10 +81,66 @@ pub fn add_superset_config( ); append_authentication_config(config, authentication_config)?; - Ok(()) } +pub(crate) fn append_celery_connection_config( + config_file: &mut Vec, + superset: &SupersetCluster, +) { + let ( + Some(additional_celery_results_backend_connection_details), + Some(celery_results_backend_connection_details), + ) = celery_results_backend_connection_details(superset) + else { + return; + }; + + let Some(celery_broker_connection_details) = celery_broker_connection_details(superset) else { + return; + }; + + let result_backend_username_env = celery_results_backend_connection_details + .username_env + .map(|env| env.name) + .unwrap_or("".to_string()); + let result_backend_password_env = celery_results_backend_connection_details + .password_env + .map(|env| env.name) + .unwrap_or("".to_string()); + let result_backend_url_template = celery_results_backend_connection_details.url_template; + let result_backend_host = additional_celery_results_backend_connection_details.host; + let result_backend_port = additional_celery_results_backend_connection_details.port; + let result_backend_db = additional_celery_results_backend_connection_details.database_id; + let broker_url_template = celery_broker_connection_details.url_template; + + let celery_config = formatdoc!( + r#" + # CELERY ASYNC + from flask_caching.backends.rediscache import RedisCache + RESULTS_BACKEND = RedisCache(host='{result_backend_host}', port={result_backend_port}, db={result_backend_db}, key_prefix='superset_results', username=os.path.expandvars('${{{result_backend_username_env}}}'), password=os.path.expandvars('${{{result_backend_password_env}}}')) + class CeleryConfig(object): + broker_url = os.path.expandvars('{broker_url_template}') + imports = ( + "superset.sql_lab", + "superset.tasks.scheduler", + ) + result_backend = os.path.expandvars('{result_backend_url_template}') + worker_prefetch_multiplier = 10 + task_acks_late = True + task_annotations = {{ + "sql_lab.get_sql_results": {{ + "rate_limit": "100/s", + }}, + }} + + CELERY_CONFIG = CeleryConfig + "#, + ); + + writeln!(config_file, "{celery_config}").expect("Writing to vec always works."); +} + fn append_authentication_config( config: &mut BTreeMap, auth_config: &SupersetClientAuthenticationDetailsResolved, diff --git a/rust/operator-binary/src/controller_commons.rs b/rust/operator-binary/src/controller_commons.rs deleted file mode 100644 index 7d16c41b..00000000 --- a/rust/operator-binary/src/controller_commons.rs +++ /dev/null @@ -1,68 +0,0 @@ -use stackable_operator::{ - builder::pod::volume::VolumeBuilder, - k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, - product_logging::{ - self, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, -}; - -use crate::crd::MAX_LOG_FILES_SIZE; - -pub const CONFIG_VOLUME_NAME: &str = "config"; -pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; -pub const LOG_VOLUME_NAME: &str = "log"; - -pub fn create_volumes( - config_map_name: &str, - log_config: Option<&ContainerLogConfig>, -) -> Vec { - let mut volumes = Vec::new(); - - volumes.push( - VolumeBuilder::new(CONFIG_VOLUME_NAME) - .with_config_map(config_map_name) - .build(), - ); - volumes.push(Volume { - name: LOG_VOLUME_NAME.into(), - empty_dir: Some(EmptyDirVolumeSource { - medium: None, - size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( - &[MAX_LOG_FILES_SIZE], - )), - }), - ..Volume::default() - }); - - if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = log_config - { - volumes.push(Volume { - name: LOG_CONFIG_VOLUME_NAME.into(), - config_map: Some(ConfigMapVolumeSource { - name: config_map.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }); - } else { - volumes.push(Volume { - name: LOG_CONFIG_VOLUME_NAME.into(), - config_map: Some(ConfigMapVolumeSource { - name: config_map_name.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }); - } - - volumes -} diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 202069d4..178746de 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -60,7 +60,10 @@ mod tests { let superset: v1alpha1::SupersetCluster = yaml_from_str_singleton_map(input).expect("illegal test input"); let merged_config = superset - .merged_config(&SupersetRole::Node, &superset.node_rolegroup_ref("default")) + .merged_config( + &SupersetRole::Node, + &superset.rolegroup_ref(&SupersetRole::Node, "default"), + ) .unwrap(); assert_eq!( diff --git a/rust/operator-binary/src/crd/databases.rs b/rust/operator-binary/src/crd/databases.rs index 5862882c..855dee77 100644 --- a/rust/operator-binary/src/crd/databases.rs +++ b/rust/operator-binary/src/crd/databases.rs @@ -3,8 +3,11 @@ use std::ops::Deref; use serde::{Deserialize, Serialize}; use stackable_operator::{ database_connections::{ - databases::postgresql::PostgresqlConnection, - drivers::sqlalchemy::{GenericSqlAlchemyDatabaseConnection, SqlAlchemyDatabaseConnection}, + databases::{postgresql::PostgresqlConnection, redis::RedisConnection}, + drivers::{ + celery::{CeleryDatabaseConnection, GenericCeleryDatabaseConnection}, + sqlalchemy::{GenericSqlAlchemyDatabaseConnection, SqlAlchemyDatabaseConnection}, + }, }, schemars::{self, JsonSchema}, }; @@ -29,3 +32,61 @@ impl Deref for MetadataDatabaseConnection { } } } + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum CeleryResultsBackendConnection { + // Docs are on the struct + Redis(RedisConnection), +} + +impl Deref for CeleryResultsBackendConnection { + type Target = dyn CeleryDatabaseConnection; + + fn deref(&self) -> &Self::Target { + match self { + Self::Redis(r) => r, + } + } +} + +impl CeleryResultsBackendConnection { + pub fn as_python_parameters(&self) -> CeleryResultsBackendConnectionDetails { + match &self { + CeleryResultsBackendConnection::Redis(redis_connection) => { + CeleryResultsBackendConnectionDetails { + host: redis_connection.host.clone(), + port: redis_connection.port, + database_id: redis_connection.database_id, + } + } + } + } +} + +pub struct CeleryResultsBackendConnectionDetails { + pub host: stackable_operator::commons::networking::HostName, + pub port: u16, + pub database_id: u16, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum CeleryBrokerConnection { + // Docs are on the struct + Redis(RedisConnection), + + // Docs are on the struct + Generic(GenericCeleryDatabaseConnection), +} + +impl Deref for CeleryBrokerConnection { + type Target = dyn CeleryDatabaseConnection; + + fn deref(&self) -> &Self::Target { + match self { + Self::Redis(r) => r, + Self::Generic(g) => g, + } + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index deee1c66..a1e780c5 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -35,8 +35,13 @@ use stackable_operator::{ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ - crd::{databases::MetadataDatabaseConnection, v1alpha1::SupersetRoleConfig}, - listener::default_listener_class, + crd::{ + databases::{ + CeleryBrokerConnection, CeleryResultsBackendConnection, MetadataDatabaseConnection, + }, + v1alpha1::SupersetRoleConfig, + }, + resources::listener::default_listener_class, }; pub mod affinity; @@ -162,6 +167,14 @@ pub mod versioned { // no doc - docs in the struct. #[serde(default, skip_serializing_if = "Option::is_none")] pub nodes: Option, + + // no doc - docs in the struct. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workers: Option, + + // no doc - docs in the struct. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub beat: Option, } // TODO: move generic version to op-rs? @@ -208,6 +221,20 @@ pub mod versioned { /// Configure the database where Superset stores all its internal metadata. pub metadata_database: MetadataDatabaseConnection, + /// Connection information for the celery backend database. + /// Only works if `workers` (and `beat`) roles are set. + /// + /// Ignored otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub celery_results_backend: Option, + + /// Connection information for the celery broker queue. + /// + /// Only works if `workers` (and `beat`) roles are set. + /// Ignored otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub celery_broker: Option, + /// The name of the Secret object containing the admin user credentials. /// Read the /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/superset/getting_started/first_steps) @@ -359,11 +386,25 @@ impl KeyValueOverridesProvider for v1alpha1::SupersetConfigOverrides { } #[derive( - Clone, Debug, Deserialize, Display, EnumIter, Eq, Hash, JsonSchema, PartialEq, Serialize, + Clone, + Debug, + Deserialize, + Display, + EnumIter, + EnumString, + Eq, + Hash, + JsonSchema, + PartialEq, + Serialize, )] pub enum SupersetRole { #[strum(serialize = "node")] Node, + #[strum(serialize = "worker")] + Worker, + #[strum(serialize = "beat")] + Beat, } impl SupersetRole { @@ -374,7 +415,16 @@ impl SupersetRole { .nodes .to_owned() .map(|node| node.role_config.listener_class), + Self::Worker | Self::Beat => None, + } + } + + pub fn roles() -> Vec { + let mut roles = vec![]; + for role in Self::iter() { + roles.push(role.to_string()) } + roles } } @@ -447,23 +497,61 @@ impl v1alpha1::SupersetConfig { pub const MAPBOX_SECRET_PROPERTY: &'static str = "mapboxSecret"; fn default_config(cluster_name: &str, role: &SupersetRole) -> v1alpha1::SupersetConfigFragment { - v1alpha1::SupersetConfigFragment { - resources: ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("300m".to_owned())), - max: Some(Quantity("1200m".to_owned())), + match role { + SupersetRole::Node => v1alpha1::SupersetConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("300m".to_owned())), + max: Some(Quantity("1200m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("2Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: v1alpha1::SupersetStorageConfigFragment {}, }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("2Gi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, + logging: product_logging::spec::default_logging(), + affinity: affinity::get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), + row_limit: None, + webserver_timeout: None, + }, + SupersetRole::Worker => v1alpha1::SupersetConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("1000m".to_owned())), + max: Some(Quantity("2000m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("4Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: v1alpha1::SupersetStorageConfigFragment {}, + }, + logging: product_logging::spec::default_logging(), + affinity: affinity::get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), + row_limit: None, + webserver_timeout: None, + }, + SupersetRole::Beat => v1alpha1::SupersetConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("100m".to_owned())), + max: Some(Quantity("500m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("1Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: v1alpha1::SupersetStorageConfigFragment {}, }, - storage: v1alpha1::SupersetStorageConfigFragment {}, + logging: product_logging::spec::default_logging(), + affinity: affinity::get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), + row_limit: None, + webserver_timeout: None, }, - logging: product_logging::spec::default_logging(), - affinity: affinity::get_affinity(cluster_name, role), - graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), - row_limit: None, - webserver_timeout: None, } } } @@ -537,6 +625,7 @@ impl v1alpha1::SupersetCluster { "{cluster_name}-{role}", cluster_name = self.name_any() )), + SupersetRole::Worker | SupersetRole::Beat => None, } } @@ -545,25 +634,26 @@ impl v1alpha1::SupersetCluster { } pub fn get_role_config(&self, role: &SupersetRole) -> Option<&SupersetRoleConfig> { - match role { - SupersetRole::Node => self.spec.nodes.as_ref().map(|c| &c.role_config), - } + self.get_role(role).as_ref().map(|c| &c.role_config) } pub fn get_role(&self, role: &SupersetRole) -> Option<&SupersetRoleType> { match role { SupersetRole::Node => self.spec.nodes.as_ref(), + SupersetRole::Worker => self.spec.workers.as_ref(), + SupersetRole::Beat => self.spec.beat.as_ref(), } } /// Metadata about a node rolegroup - pub fn node_rolegroup_ref( + pub fn rolegroup_ref( &self, + role: &SupersetRole, group_name: impl Into, ) -> RoleGroupRef { RoleGroupRef { cluster: ObjectRef::from_obj(self), - role: SupersetRole::Node.to_string(), + role: role.to_string(), role_group: group_name.into(), } } @@ -585,12 +675,10 @@ impl v1alpha1::SupersetCluster { // Initialize the result with all default values as baseline let conf_defaults = v1alpha1::SupersetConfig::default_config(&self.name_any(), role); - let role = match role { - SupersetRole::Node => self.spec.nodes.as_ref().context(UnknownSupersetRoleSnafu { - role: role.to_string(), - roles: vec![role.to_string()], - })?, - }; + let role = self.get_role(role).context(UnknownSupersetRoleSnafu { + role: role.to_string(), + roles: SupersetRole::roles(), + })?; // Retrieve role resource config let mut conf_role = role.config.config.to_owned(); diff --git a/rust/operator-binary/src/druid_connection_controller.rs b/rust/operator-binary/src/druid_connection_controller.rs index 5d2e6305..3acacd09 100644 --- a/rust/operator-binary/src/druid_connection_controller.rs +++ b/rust/operator-binary/src/druid_connection_controller.rs @@ -31,9 +31,9 @@ use crate::{ crd::{ INTERNAL_SECRET_SECRET_KEY, PYTHONPATH, SUPERSET_CONFIG_FILENAME, druidconnection, v1alpha1, }, - rbac, + operations::job_state::{JobState, get_job_state}, + resources::rbac, superset_controller::CONTAINER_IMAGE_BASE_NAME, - util::{JobState, get_job_state}, }; pub const DRUID_CONNECTION_CONTROLLER_NAME: &str = "druid-connection"; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index e52f3acf..cf96dc46 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -12,7 +12,7 @@ use stackable_operator::{ crd::authentication::core, eos::EndOfSupportChecker, k8s_openapi::api::{ - apps::v1::StatefulSet, + apps::v1::{Deployment, StatefulSet}, batch::v1::Job, core::v1::{ConfigMap, Service}, }, @@ -44,18 +44,12 @@ use crate::{ }; mod authorization; -mod commands; mod config; -mod controller_commons; mod crd; mod druid_connection_controller; -mod listener; mod operations; -mod product_logging; -mod rbac; -mod service; +mod resources; mod superset_controller; -mod util; mod webhooks; mod built_info { @@ -158,6 +152,11 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), ) + // Required for workers and beat. + .owns( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ) .watches( client.get_api::>(&()), watcher::Config::default(), diff --git a/rust/operator-binary/src/util.rs b/rust/operator-binary/src/operations/job_state.rs similarity index 51% rename from rust/operator-binary/src/util.rs rename to rust/operator-binary/src/operations/job_state.rs index 0ddf70e7..7b3defea 100644 --- a/rust/operator-binary/src/util.rs +++ b/rust/operator-binary/src/operations/job_state.rs @@ -1,6 +1,4 @@ -use stackable_operator::{k8s_openapi::api::batch::v1::Job, kvp::ObjectLabels}; - -use crate::{OPERATOR_NAME, crd::APP_NAME}; +use stackable_operator::k8s_openapi::api::batch::v1::Job; pub enum JobState { InProgress, @@ -29,22 +27,3 @@ pub fn get_job_state(job: &Job) -> JobState { JobState::InProgress } } - -/// Creates recommended `ObjectLabels` to be used in deployed resources -pub fn build_recommended_labels<'a, T>( - owner: &'a T, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -} diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs index 92ca2ec7..8da9c1c5 100644 --- a/rust/operator-binary/src/operations/mod.rs +++ b/rust/operator-binary/src/operations/mod.rs @@ -1,2 +1,3 @@ pub mod graceful_shutdown; +pub mod job_state; pub mod pdb; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index e8b02fd0..15275de2 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -36,6 +36,8 @@ pub async fn add_pdbs( } let max_unavailable = pdb.max_unavailable.unwrap_or(match role { SupersetRole::Node => max_unavailable_nodes(), + SupersetRole::Worker => max_unavailable_workers(), + SupersetRole::Beat => max_unavailable_beat(), }); let pdb = PodDisruptionBudgetBuilder::new_with_role( superset, @@ -61,3 +63,11 @@ pub async fn add_pdbs( fn max_unavailable_nodes() -> u16 { 1 } + +fn max_unavailable_workers() -> u16 { + 1 +} + +fn max_unavailable_beat() -> u16 { + 1 +} diff --git a/rust/operator-binary/src/resources/configmap.rs b/rust/operator-binary/src/resources/configmap.rs new file mode 100644 index 00000000..f5c93eeb --- /dev/null +++ b/rust/operator-binary/src/resources/configmap.rs @@ -0,0 +1,171 @@ +use std::{ + collections::{BTreeMap, HashMap}, + io::Write, +}; + +use product_config::{ + flask_app_config_writer::{self}, + types::PropertyNameKind, +}; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::ConfigMap, + product_config_utils::{CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY}, + product_logging::spec::Logging, + role_utils::RoleGroupRef, +}; + +use crate::{ + authorization::opa::{OPA_IMPORTS, SupersetOpaConfigResolved}, + config::{self, product_logging::extend_config_map_with_log_config, superset::PYTHON_IMPORTS}, + crd::{ + SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, + authentication::SupersetClientAuthenticationDetailsResolved, + v1alpha1::{Container, SupersetCluster}, + }, + resources::build_recommended_labels, + superset_controller::SUPERSET_CONTROLLER_NAME, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to add Superset config settings"))] + AddSupersetConfig { source: config::superset::Error }, + + #[snafu(display( + "failed to write to String (Vec to be precise) containing superset config" + ))] + WriteToConfigFileString { source: std::io::Error }, + + #[snafu(display("failed to build Metadata"))] + BuildMetadata { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build config file for {rolegroup}"))] + BuildRoleGroupConfigFile { + source: flask_app_config_writer::FlaskAppConfigWriterError, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to build ConfigMap for {rolegroup}"))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] +pub fn build_rolegroup_config_map( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, + rolegroup_config: &HashMap>, + authentication_config: &SupersetClientAuthenticationDetailsResolved, + superset_opa_config: &Option, + logging: &Logging, +) -> Result { + let mut config_properties = BTreeMap::new(); + let mut imports = PYTHON_IMPORTS.to_vec(); + // TODO: this is true per default for versions 3.0.0 and up. + // We deactivate it here to keep existing functionality. + // However this is a security issue and should be configured properly + // Issue: https://github.com/stackabletech/superset-operator/issues/416 + config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); + + config::superset::add_superset_config(&mut config_properties, superset, authentication_config) + .context(AddSupersetConfigSnafu)?; + + // Adding opa configuration properties to config_properties. + // This will be injected as key/value pair in superset_config.py + if let Some(opa_config) = superset_opa_config { + // If opa role mapping is configured, insert CustomOpaSecurityManager import + imports.extend(OPA_IMPORTS); + + config_properties.extend(opa_config.as_config()); + } + + // The order here should be kept in order to preserve overrides. + // No properties should be added after this extend. + config_properties.extend( + rolegroup_config + .get(&PropertyNameKind::File( + SUPERSET_CONFIG_FILENAME.to_string(), + )) + .cloned() + .unwrap_or_default(), + ); + + let mut config_file = Vec::new(); + + // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{header}").context(WriteToConfigFileStringSnafu)?; + } + let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + + flask_app_config_writer::write::( + &mut config_file, + config_properties.iter(), + &imports, + ) + .with_context(|_| BuildRoleGroupConfigFileSnafu { + rolegroup: rolegroup.clone(), + })?; + + // We have to add a python class (no key) and cannot use the superset::config machinery. + config::superset::append_celery_connection_config(&mut config_file, superset); + + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{footer}").context(WriteToConfigFileStringSnafu)?; + } + + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(BuildMetadataSnafu)? + .build(), + ) + .add_data( + SUPERSET_CONFIG_FILENAME, + String::from_utf8(config_file).unwrap(), + ); + + extend_config_map_with_log_config( + rolegroup, + logging, + &Container::Superset, + &Container::Vector, + &mut cm_builder, + ); + + cm_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + }) +} diff --git a/rust/operator-binary/src/resources/deployment.rs b/rust/operator-binary/src/resources/deployment.rs new file mode 100644 index 00000000..cd9bf33a --- /dev/null +++ b/rust/operator-binary/src/resources/deployment.rs @@ -0,0 +1,639 @@ +use std::collections::{BTreeMap, HashMap}; + +use indoc::formatdoc; +use product_config::types::PropertyNameKind; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + }, + }, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{Deployment, DeploymentSpec}, + core::v1::{ExecAction, Probe}, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kvp::{Label, Labels}, + product_logging::{ + self, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, + }, + role_utils::RoleGroupRef, + utils::COMMON_BASH_TRAP_FUNCTIONS, +}; + +use crate::{ + config::product_logging::LOG_CONFIG_FILE, + crd::{ + APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, + SupersetConfigOptions, SupersetRole, + v1alpha1::{Container, SupersetCluster, SupersetConfig}, + }, + operations::graceful_shutdown::add_graceful_shutdown_config, + resources::{ + CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME, build_recommended_labels, + }, + superset_controller::SUPERSET_CONTROLLER_NAME, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object defines no '{role}' role"))] + MissingRole { role: String }, + + #[snafu(display("object defines no '{role}' rolegroup"))] + MissingRoleGroup { role: String }, + + #[snafu(display("invalid container name"))] + InvalidContainerName { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display( + "failed to get the {SUPERSET_CONFIG_FILENAME} file from node or product config" + ))] + MissingSupersetConfigInNodeConfig, + + #[snafu(display("failed to get {timeout} from {SUPERSET_CONFIG_FILENAME} file. It should be set in the product config or by user input", timeout = SupersetConfigOptions::SupersetWebserverTimeout))] + MissingWebServerTimeoutInSupersetConfig, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { + source: product_logging::framework::LoggingError, + }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`Deployment`] runs the rolegroup, as configured by the administrator. +pub fn build_worker_rolegroup_deployment( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + superset_role: &SupersetRole, + rolegroup_ref: &RoleGroupRef, + node_config: &HashMap>, + sa_name: &str, + merged_config: &SupersetConfig, +) -> Result { + let role = superset + .get_role(superset_role) + .with_context(|| MissingRoleSnafu { + role: superset_role.to_string(), + })?; + let role_group = role + .role_groups + .get(&rolegroup_ref.role_group) + .with_context(|| MissingRoleGroupSnafu { + role: superset_role.to_string(), + })?; + + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .build(); + + let mut pb = &mut PodBuilder::new(); + + pb = pb + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .security_context( + PodSecurityContextBuilder::new() + .fs_group(1000) // Needed for secret-operator + .build(), + ) + .affinity(&merged_config.affinity) + .service_account_name(sa_name); + + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + + let metadata_database_connection_details = + super::metadata_database_connection_details(superset); + let celery_results_backend_connection_details = + super::celery_results_backend_connection_details(superset); + let celery_broker_connection_details = super::celery_broker_connection_details(superset); + + metadata_database_connection_details.add_to_container(&mut superset_cb); + if let (_, Some(celery_results_backend_connection_details)) = + &celery_results_backend_connection_details + { + celery_results_backend_connection_details.add_to_container(&mut superset_cb); + } + if let Some(celery_broker_connection_details) = celery_broker_connection_details { + celery_broker_connection_details.add_to_container(&mut superset_cb); + } + + for (name, value) in node_config + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default() + { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret( + "MAPBOX_API_KEY", + &value, + "connections.mapboxApiKey", + ); + } else { + superset_cb.add_env_var(name, value); + }; + } + + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + crate::crd::INTERNAL_SECRET_SECRET_KEY, + ); + + let secret = &superset.spec.cluster_config.credentials_secret_name; + + superset_cb + .image_from_product_image(resolved_product_image) + .add_container_port("http", APP_PORT.into()) + .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") + .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") + .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") + .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") + .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") + // Needed by the `containerdebug` process to log it's tracing information to. + .add_env_var( + "CONTAINERDEBUG_LOG_DIRECTORY", + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ) + .add_env_var("SSL_CERT_DIR", "/stackable/certs/") + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + mkdir --parents {PYTHONPATH} + cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} + cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} + + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + + celery --app=superset.tasks.celery_app:app worker --task-events & + + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]) + .liveness_probe(Probe { + exec: Some(ExecAction { + command: Some(vec![ + "celery --app=superset.tasks.celery_app:app inspect ping -d celery@$HOSTNAME" + .to_string(), + ]), + }), + initial_delay_seconds: Some(30), + period_seconds: Some(30), + timeout_seconds: Some(30), + failure_threshold: Some(3), + ..Default::default() + }) + .resources(merged_config.resources.clone().into()); + + pb.add_container(superset_cb.build()); + add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; + + let metrics_container = ContainerBuilder::new("metrics") + .context(InvalidContainerNameSnafu)? + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + prepare_signal_handlers + /stackable/statsd_exporter & + wait_for_termination $! + "}]) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("64Mi") + .with_memory_limit("64Mi") + .build(), + ) + .build(); + + pb.add_volumes(crate::resources::create_volumes( + &rolegroup_ref.object_name(), + merged_config.logging.containers.get(&Container::Superset), + )) + .context(AddVolumeSnafu)?; + pb.add_container(metrics_container); + + if merged_config.logging.enable_vector_agent { + match &superset + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some(vector_aggregator_config_map_name) => { + pb.add_container( + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(role.config.pod_overrides.clone()); + pod_template.merge_from(role_group.config.pod_overrides.clone()); + + Ok(Deployment { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("restarter.stackable.tech/enabled", "true")) + .context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(DeploymentSpec { + replicas: role_group.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + superset, + APP_NAME, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + template: pod_template, + ..DeploymentSpec::default() + }), + status: None, + }) +} + +/// The rolegroup [`Deployment`] runs the rolegroup, as configured by the administrator. +pub fn build_beat_rolegroup_deployment( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + superset_role: &SupersetRole, + rolegroup_ref: &RoleGroupRef, + node_config: &HashMap>, + sa_name: &str, + merged_config: &SupersetConfig, +) -> Result { + let role = superset + .get_role(superset_role) + .with_context(|| MissingRoleSnafu { + role: superset_role.to_string(), + })?; + let role_group = role + .role_groups + .get(&rolegroup_ref.role_group) + .with_context(|| MissingRoleGroupSnafu { + role: superset_role.to_string(), + })?; + + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .build(); + + let mut pb = &mut PodBuilder::new(); + + pb = pb + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .security_context( + PodSecurityContextBuilder::new() + .fs_group(1000) // Needed for secret-operator + .build(), + ) + .affinity(&merged_config.affinity) + .service_account_name(sa_name); + + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + + let metadata_database_connection_details = + super::metadata_database_connection_details(superset); + let celery_results_backend_connection_details = + super::celery_results_backend_connection_details(superset); + let celery_broker_connection_details = super::celery_broker_connection_details(superset); + + metadata_database_connection_details.add_to_container(&mut superset_cb); + if let (_, Some(celery_results_backend_connection_details)) = + &celery_results_backend_connection_details + { + celery_results_backend_connection_details.add_to_container(&mut superset_cb); + } + if let Some(celery_broker_connection_details) = celery_broker_connection_details { + celery_broker_connection_details.add_to_container(&mut superset_cb); + } + + for (name, value) in node_config + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default() + { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret( + "MAPBOX_API_KEY", + &value, + "connections.mapboxApiKey", + ); + } else { + superset_cb.add_env_var(name, value); + }; + } + + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + crate::crd::INTERNAL_SECRET_SECRET_KEY, + ); + + let secret = &superset.spec.cluster_config.credentials_secret_name; + + superset_cb + .image_from_product_image(resolved_product_image) + .add_container_port("http", APP_PORT.into()) + .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") + .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") + .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") + .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") + .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") + // Needed by the `containerdebug` process to log it's tracing information to. + .add_env_var( + "CONTAINERDEBUG_LOG_DIRECTORY", + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ) + .add_env_var("SSL_CERT_DIR", "/stackable/certs/") + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + mkdir --parents {PYTHONPATH} + cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} + cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} + + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + + celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid & + + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]) + .liveness_probe(Probe { + exec: Some(ExecAction { + command: Some(vec![ + "[ -f /tmp/celerybeat.pid ] && kill -0 $(cat /tmp/celerybeat.pid)".to_string(), + ]), + }), + initial_delay_seconds: Some(30), + period_seconds: Some(30), + timeout_seconds: Some(30), + failure_threshold: Some(3), + ..Default::default() + }) + .resources(merged_config.resources.clone().into()); + + pb.add_container(superset_cb.build()); + add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; + + let metrics_container = ContainerBuilder::new("metrics") + .context(InvalidContainerNameSnafu)? + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + prepare_signal_handlers + /stackable/statsd_exporter & + wait_for_termination $! + "}]) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("64Mi") + .with_memory_limit("64Mi") + .build(), + ) + .build(); + + pb.add_volumes(crate::resources::create_volumes( + &rolegroup_ref.object_name(), + merged_config.logging.containers.get(&Container::Superset), + )) + .context(AddVolumeSnafu)?; + pb.add_container(metrics_container); + + if merged_config.logging.enable_vector_agent { + match &superset + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some(vector_aggregator_config_map_name) => { + pb.add_container( + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(role.config.pod_overrides.clone()); + pod_template.merge_from(role_group.config.pod_overrides.clone()); + + let replicas = if let Some(replicas) = role_group.replicas { + if replicas > 1 { + tracing::warn! {"replicas for role `beat` set to greater `1`. Multiple beat instances are not allowed. Setting to `1` replica."} + } + 1 + } else { + 0 + }; + + Ok(Deployment { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("restarter.stackable.tech/enabled", "true")) + .context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(DeploymentSpec { + // Beat should always only be one Beat instance at a time. + // We ignore values > 1, 0 is a possible value still. + replicas: Some(replicas), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + superset, + APP_NAME, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + template: pod_template, + ..DeploymentSpec::default() + }), + status: None, + }) +} diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/resources/listener.rs similarity index 100% rename from rust/operator-binary/src/listener.rs rename to rust/operator-binary/src/resources/listener.rs diff --git a/rust/operator-binary/src/resources/mod.rs b/rust/operator-binary/src/resources/mod.rs new file mode 100644 index 00000000..3e94d08e --- /dev/null +++ b/rust/operator-binary/src/resources/mod.rs @@ -0,0 +1,164 @@ +use stackable_operator::{ + builder::pod::volume::VolumeBuilder, + database_connections::{ + TemplatingMechanism, + drivers::{ + celery::CeleryDatabaseConnectionDetails, + sqlalchemy::SqlAlchemyDatabaseConnectionDetails, + }, + }, + k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, + kvp::ObjectLabels, + product_logging::{ + self, + spec::{ + ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, + }, + }, +}; + +use crate::{ + OPERATOR_NAME, + crd::{APP_NAME, databases::CeleryResultsBackendConnectionDetails}, + v1alpha1::SupersetCluster, +}; + +pub mod configmap; +pub mod deployment; +pub mod listener; +pub mod rbac; +pub mod service; +pub mod statefulset; + +use crate::crd::MAX_LOG_FILES_SIZE; + +pub const CONFIG_VOLUME_NAME: &str = "config"; +pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; +pub const LOG_VOLUME_NAME: &str = "log"; + +/// Creates recommended `ObjectLabels` to be used in deployed resources +pub fn build_recommended_labels<'a, T>( + owner: &'a T, + controller_name: &'a str, + app_version: &'a str, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, T> { + ObjectLabels { + owner, + app_name: APP_NAME, + app_version, + operator_name: OPERATOR_NAME, + controller_name, + role, + role_group, + } +} + +pub(crate) fn create_volumes( + config_map_name: &str, + log_config: Option<&ContainerLogConfig>, +) -> Vec { + let mut volumes = Vec::new(); + + volumes.push( + VolumeBuilder::new(CONFIG_VOLUME_NAME) + .with_config_map(config_map_name) + .build(), + ); + volumes.push(Volume { + name: LOG_VOLUME_NAME.into(), + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_LOG_FILES_SIZE], + )), + }), + ..Volume::default() + }); + + if let Some(ContainerLogConfig { + choice: + Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + })), + }) = log_config + { + volumes.push(Volume { + name: LOG_CONFIG_VOLUME_NAME.into(), + config_map: Some(ConfigMapVolumeSource { + name: config_map.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }); + } else { + volumes.push(Volume { + name: LOG_CONFIG_VOLUME_NAME.into(), + config_map: Some(ConfigMapVolumeSource { + name: config_map_name.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }); + } + + volumes +} + +pub(crate) fn metadata_database_connection_details( + superset: &SupersetCluster, +) -> SqlAlchemyDatabaseConnectionDetails { + superset + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating( + "METADATA", + &TemplatingMechanism::BashEnvSubstitution, + ) +} + +pub(crate) fn celery_results_backend_connection_details( + superset: &SupersetCluster, +) -> ( + Option, + Option, +) { + ( + superset + .spec + .cluster_config + .celery_results_backend + .as_ref() + .map(|backend| backend.as_python_parameters()), + superset + .spec + .cluster_config + .celery_results_backend + .as_ref() + .map(|backend| { + backend.celery_connection_details_with_templating( + "CELERY_RESULTS_BACKEND", + &TemplatingMechanism::BashEnvSubstitution, + ) + }), + ) +} + +pub(crate) fn celery_broker_connection_details( + superset: &SupersetCluster, +) -> Option { + superset + .spec + .cluster_config + .celery_broker + .as_ref() + .map(|broker| { + broker.celery_connection_details_with_templating( + "CELERY_BROKER", + &TemplatingMechanism::BashEnvSubstitution, + ) + }) +} diff --git a/rust/operator-binary/src/rbac.rs b/rust/operator-binary/src/resources/rbac.rs similarity index 100% rename from rust/operator-binary/src/rbac.rs rename to rust/operator-binary/src/resources/rbac.rs diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/resources/service.rs similarity index 99% rename from rust/operator-binary/src/service.rs rename to rust/operator-binary/src/resources/service.rs index d7c51c73..b8e2e042 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/resources/service.rs @@ -9,9 +9,10 @@ use stackable_operator::{ use crate::{ crd::{APP_NAME, APP_PORT, APP_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}, + resources::build_recommended_labels, superset_controller::SUPERSET_CONTROLLER_NAME, - util::build_recommended_labels, }; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("object is missing metadata to build owner reference"))] diff --git a/rust/operator-binary/src/resources/statefulset.rs b/rust/operator-binary/src/resources/statefulset.rs new file mode 100644 index 00000000..2c49032c --- /dev/null +++ b/rust/operator-binary/src/resources/statefulset.rs @@ -0,0 +1,561 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use indoc::formatdoc; +use product_config::types::PropertyNameKind; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, + container::ContainerBuilder, + probe::ProbeBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, + }, + }, + }, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{StatefulSet, StatefulSetSpec}, + core::v1::EnvVar, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kvp::{Label, Labels}, + product_logging::{ + self, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, + }, + role_utils::RoleGroupRef, + shared::time::Duration, + utils::COMMON_BASH_TRAP_FUNCTIONS, +}; + +use crate::{ + config::{commands::add_cert_to_python_certifi_command, product_logging::LOG_CONFIG_FILE}, + crd::{ + APP_NAME, APP_PORT, METRICS_PORT, METRICS_PORT_NAME, PYTHONPATH, STACKABLE_CONFIG_DIR, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SUPERSET_CONFIG_FILENAME, + SupersetConfigOptions, SupersetRole, + authentication::{ + SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, + }, + v1alpha1::{Container, SupersetCluster, SupersetConfig}, + }, + operations::graceful_shutdown::add_graceful_shutdown_config, + resources::{ + CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME, build_recommended_labels, + listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, + }, + superset_controller::SUPERSET_CONTROLLER_NAME, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object defines no '{role}' role"))] + MissingRole { role: String }, + + #[snafu(display("object defines no '{role}' rolegroup"))] + MissingRoleGroup { role: String }, + + #[snafu(display("invalid container name"))] + InvalidContainerName { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display( + "failed to get the {SUPERSET_CONFIG_FILENAME} file from node or product config" + ))] + MissingSupersetConfigInNodeConfig, + + #[snafu(display("failed to get {timeout} from {SUPERSET_CONFIG_FILENAME} file. It should be set in the product config or by user input", timeout = SupersetConfigOptions::SupersetWebserverTimeout))] + MissingWebServerTimeoutInSupersetConfig, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { + source: product_logging::framework::LoggingError, + }, + + #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] + AddLdapVolumesAndVolumeMounts { + source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, + }, + + #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] + AddTlsVolumesAndVolumeMounts { + source: stackable_operator::commons::tls_verification::TlsClientDetailsError, + }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. +#[allow(clippy::too_many_arguments)] +pub fn build_server_rolegroup_statefulset( + superset: &SupersetCluster, + resolved_product_image: &ResolvedProductImage, + superset_role: &SupersetRole, + rolegroup_ref: &RoleGroupRef, + node_config: &HashMap>, + authentication_config: &SupersetClientAuthenticationDetailsResolved, + sa_name: &str, + merged_config: &SupersetConfig, +) -> Result { + let role = superset + .get_role(superset_role) + .with_context(|| MissingRoleSnafu { + role: superset_role.to_string(), + })?; + let role_group = role + .role_groups + .get(&rolegroup_ref.role_group) + .with_context(|| MissingRoleGroupSnafu { + role: superset_role.to_string(), + })?; + + let recommended_object_labels = build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .build(); + + let mut pb = &mut PodBuilder::new(); + + pb = pb + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .security_context( + PodSecurityContextBuilder::new() + .fs_group(1000) // Needed for secret-operator + .build(), + ) + .affinity(&merged_config.affinity) + .service_account_name(sa_name); + + let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) + .context(InvalidContainerNameSnafu)?; + + let metadata_database_connection_details = + super::metadata_database_connection_details(superset); + let celery_results_backend_connection_details = + super::celery_results_backend_connection_details(superset); + let celery_broker_connection_details = super::celery_broker_connection_details(superset); + + metadata_database_connection_details.add_to_container(&mut superset_cb); + if let (_, Some(celery_results_backend_connection_details)) = + &celery_results_backend_connection_details + { + celery_results_backend_connection_details.add_to_container(&mut superset_cb); + } + if let Some(celery_broker_connection_details) = celery_broker_connection_details { + celery_broker_connection_details.add_to_container(&mut superset_cb); + } + + for (name, value) in node_config + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default() + { + if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { + superset_cb.add_env_var_from_secret( + "MAPBOX_API_KEY", + &value, + "connections.mapboxApiKey", + ); + } else { + superset_cb.add_env_var(name, value); + }; + } + + // SECRET_KEY from auto-generated secret + superset_cb.add_env_var_from_secret( + "SECRET_KEY", + superset.shared_secret_key_secret_name(), + crate::crd::INTERNAL_SECRET_SECRET_KEY, + ); + + add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; + + let webserver_timeout = node_config + .get(&PropertyNameKind::File( + SUPERSET_CONFIG_FILENAME.to_string(), + )) + .context(MissingSupersetConfigInNodeConfigSnafu)? + .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) + .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; + + let secret = &superset.spec.cluster_config.credentials_secret_name; + + superset_cb + .image_from_product_image(resolved_product_image) + .add_container_port("http", APP_PORT.into()) + .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR).context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR).context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR).context(AddVolumeMountSnafu)? + .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") + .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") + .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") + .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") + .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") + // Needed by the `containerdebug` process to log it's tracing information to. + .add_env_var("CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug")) + .add_env_var("SSL_CERT_DIR", "/stackable/certs/") + .add_env_vars(authentication_env_vars(authentication_config)) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + mkdir --parents {PYTHONPATH} + cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} + cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} + + {auth_commands} + + superset db upgrade + set +x + echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' + superset fab create-admin --username \"$ADMIN_USERNAME\" --firstname \"$ADMIN_FIRSTNAME\" --lastname \"$ADMIN_LASTNAME\" --email \"$ADMIN_EMAIL\" --password \"$ADMIN_PASSWORD\" + set -x + superset init + + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + gunicorn --bind 0.0.0.0:${{SUPERSET_PORT}} --worker-class gthread --threads 20 --timeout {webserver_timeout} --limit-request-line 0 --limit-request-field_size 0 'superset.app:create_app()' & + wait_for_termination $! + + {create_vector_shutdown_file_command} + ", + auth_commands = authentication_start_commands(authentication_config), + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]) + .resources(merged_config.resources.clone().into()); + add_superset_container_probes(&mut superset_cb); + + // listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses and + // that it is possible to connect to a consistent address + let pvcs = if let Some(group_listener_name) = superset.group_listener_name(superset_role) { + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(group_listener_name), + &unversioned_recommended_labels, + ) + .build_pvc(LISTENER_VOLUME_NAME.to_owned()) + .context(BuildListenerVolumeSnafu)?; + Some(vec![pvc]) + } else { + None + }; + + superset_cb + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + + pb.add_container(superset_cb.build()); + add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; + + let metrics_container = ContainerBuilder::new("metrics") + .context(InvalidContainerNameSnafu)? + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + + prepare_signal_handlers + /stackable/statsd_exporter & + wait_for_termination $! + "}]) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("64Mi") + .with_memory_limit("64Mi") + .build(), + ) + .build(); + + pb.add_volumes(crate::resources::create_volumes( + &rolegroup_ref.object_name(), + merged_config.logging.containers.get(&Container::Superset), + )) + .context(AddVolumeSnafu)?; + pb.add_container(metrics_container); + + if merged_config.logging.enable_vector_agent { + match &superset + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some(vector_aggregator_config_map_name) => { + pb.add_container( + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(role.config.pod_overrides.clone()); + pod_template.merge_from(role_group.config.pod_overrides.clone()); + + Ok(StatefulSet { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(superset) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(superset, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&recommended_object_labels) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("restarter.stackable.tech/enabled", "true")) + .context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(StatefulSetSpec { + // Set to `OrderedReady`, to make sure Pods start after another and the init commands don't run in parallel + pod_management_policy: Some("OrderedReady".to_string()), + replicas: role_group.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + superset, + APP_NAME, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), + template: pod_template, + volume_claim_templates: pvcs, + ..StatefulSetSpec::default() + }), + status: None, + }) +} + +fn add_superset_container_probes(superset_cb: &mut ContainerBuilder) { + let common = + ProbeBuilder::http_get_port_scheme_path(APP_PORT, None, Some("/health".to_owned())) + .with_period(Duration::from_secs(5)); + + superset_cb.startup_probe( + common + .clone() + .with_failure_threshold_duration(Duration::from_minutes_unchecked(10)) + .expect("const period is non-zero") + .build() + .expect("const duration does not overflow"), + ); + + // Remove it from the Service immediately + superset_cb.readiness_probe( + common + .clone() + .build() + .expect("const duration does not overflow"), + ); + // But only restart it after 3 failures + superset_cb.liveness_probe( + common + .with_failure_threshold(3) + .build() + .expect("const duration does not overflow"), + ); +} + +fn add_authentication_volumes_and_volume_mounts( + auth_config: &SupersetClientAuthenticationDetailsResolved, + cb: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + // Different authentication entries can reference the same secret + // class or TLS certificate. It must be ensured that the volumes + // and volume mounts are only added once in such a case. + + let mut ldap_authentication_providers = BTreeSet::new(); + let mut tls_client_credentials = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + SupersetAuthenticationClassResolved::Ldap { provider } => { + ldap_authentication_providers.insert(provider); + } + SupersetAuthenticationClassResolved::Oidc { provider, .. } => { + tls_client_credentials.insert(&provider.tls); + } + } + } + + for provider in ldap_authentication_providers { + provider + .add_volumes_and_mounts(pb, vec![cb]) + .context(AddLdapVolumesAndVolumeMountsSnafu)?; + } + + for tls in tls_client_credentials { + tls.add_volumes_and_mounts(pb, vec![cb]) + .context(AddTlsVolumesAndVolumeMountsSnafu)?; + } + + Ok(()) +} + +fn authentication_env_vars( + auth_config: &SupersetClientAuthenticationDetailsResolved, +) -> Vec { + // Different OIDC authentication entries can reference the same + // client secret. It must be ensured that the env variables are only + // added once in such a case. + + let mut oidc_client_credentials_secrets = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + SupersetAuthenticationClassResolved::Ldap { .. } => {} + SupersetAuthenticationClassResolved::Oidc { + client_auth_options: oidc, + .. + } => { + oidc_client_credentials_secrets + .insert(oidc.client_credentials_secret_ref.to_owned()); + } + } + } + + oidc_client_credentials_secrets + .iter() + .cloned() + .flat_map(stackable_operator::crd::authentication::oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts) + .collect() +} + +fn authentication_start_commands( + auth_config: &SupersetClientAuthenticationDetailsResolved, +) -> String { + let mut commands = Vec::new(); + + let mut tls_client_credentials = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + SupersetAuthenticationClassResolved::Oidc { provider, .. } => { + tls_client_credentials.insert(&provider.tls); + + // WebPKI will be handled implicitly + } + SupersetAuthenticationClassResolved::Ldap { .. } => {} + } + } + + for tls in tls_client_credentials { + commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { + add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) + })); + } + + commands + .iter() + .flatten() + .cloned() + .collect::>() + .join("\n") +} diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 6449abe0..89f03c8a 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -1,36 +1,10 @@ //! Ensures that `Pod`s are configured and running for each [`SupersetCluster`] -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet, HashMap}, - io::Write, - sync::Arc, -}; +use std::{collections::HashMap, str::FromStr, sync::Arc}; use const_format::concatcp; -use indoc::formatdoc; -use product_config::{ - ProductConfigManager, - flask_app_config_writer::{self, FlaskAppConfigWriterError}, - types::PropertyNameKind, -}; -use snafu::{OptionExt, ResultExt, Snafu}; +use product_config::{ProductConfigManager, types::PropertyNameKind}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - configmap::ConfigMapBuilder, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, - container::ContainerBuilder, - probe::ProbeBuilder, - resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, - volume::{ - ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, - ListenerReference, - }, - }, - }, cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ @@ -38,64 +12,39 @@ use stackable_operator::{ random_secret_creation, rbac::build_rbac_resources, }, - crd::authentication::oidc, - database_connections::TemplatingMechanism, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ConfigMap, EnvVar}, - }, - apimachinery::pkg::apis::meta::v1::LabelSelector, - }, kube::{ Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Label, Labels}, logging::controller::ReconcilerError, - product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, - transform_all_roles_to_config, validate_all_roles_and_groups_config, - }, - product_logging::{ - self, - framework::{ - LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command, - }, - spec::Logging, - }, + product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::{GenericRoleConfig, RoleGroupRef}, shared::time::Duration, status::condition::{ - compute_conditions, operations::ClusterOperationsConditionBuilder, - statefulset::StatefulSetConditionBuilder, + compute_conditions, deployment::DeploymentConditionBuilder, + operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use strum::{EnumDiscriminants, IntoStaticStr}; +use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; use crate::{ OPERATOR_NAME, - authorization::opa::{OPA_IMPORTS, SupersetOpaConfigResolved}, - commands::add_cert_to_python_certifi_command, - config::{self, PYTHON_IMPORTS}, - controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + authorization::opa::SupersetOpaConfigResolved, crd::{ - APP_NAME, APP_PORT, INTERNAL_SECRET_SECRET_KEY, METRICS_PORT, METRICS_PORT_NAME, - PYTHONPATH, STACKABLE_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - SUPERSET_CONFIG_FILENAME, SupersetConfigOptions, SupersetRole, - authentication::{ - SupersetAuthenticationClassResolved, SupersetClientAuthenticationDetailsResolved, - }, - v1alpha1::{Container, SupersetCluster, SupersetClusterStatus, SupersetConfig}, + APP_NAME, INTERNAL_SECRET_SECRET_KEY, SUPERSET_CONFIG_FILENAME, SupersetRole, + authentication::SupersetClientAuthenticationDetailsResolved, + v1alpha1::{SupersetCluster, SupersetClusterStatus}, + }, + operations::pdb::add_pdbs, + resources::{ + build_recommended_labels, + configmap::build_rolegroup_config_map, + deployment::{build_beat_rolegroup_deployment, build_worker_rolegroup_deployment}, + listener::build_group_listener, + service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, + statefulset::build_server_rolegroup_statefulset, }, - listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener}, - operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::{LOG_CONFIG_FILE, extend_config_map_with_log_config}, - service::{build_node_rolegroup_headless_service, build_node_rolegroup_metrics_service}, - util::build_recommended_labels, }; pub const SUPERSET_CONTROLLER_NAME: &str = "supersetcluster"; @@ -113,16 +62,11 @@ pub struct Ctx { #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("object defines no node role"))] - NoNodeRole, + #[snafu(display("object defines no '{role}' role"))] + MissingRole { role: String }, - #[snafu(display("object defines no node role-group"))] - NoNodeRoleGroup, - - #[snafu(display("invalid container name"))] - InvalidContainerName { - source: stackable_operator::builder::pod::container::Error, - }, + #[snafu(display("failed to parse role: {source}"))] + ParseRole { source: strum::ParseError }, #[snafu(display("failed to create cluster resources"))] CreateClusterResources { @@ -140,18 +84,6 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to build config file for {rolegroup}"))] - BuildRoleGroupConfigFile { - source: FlaskAppConfigWriterError, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("failed to build ConfigMap for {rolegroup}"))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to apply ConfigMap for {rolegroup}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, @@ -164,6 +96,12 @@ pub enum Error { rolegroup: RoleGroupRef, }, + #[snafu(display("failed to apply Deployment for {rolegroup}"))] + ApplyRoleGroupDeployment { + source: stackable_operator::cluster_resources::Error, + rolegroup: RoleGroupRef, + }, + #[snafu(display("failed to generate product config"))] GenerateProductConfig { source: stackable_operator::product_config_utils::Error, @@ -174,36 +112,14 @@ pub enum Error { source: stackable_operator::product_config_utils::Error, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to apply authentication configuration"))] InvalidAuthenticationConfig { source: crate::crd::authentication::Error, }, - #[snafu(display( - "failed to get the {SUPERSET_CONFIG_FILENAME} file from node or product config" - ))] - MissingSupersetConfigInNodeConfig, - - #[snafu(display("failed to get {timeout} from {SUPERSET_CONFIG_FILENAME} file. It should be set in the product config or by user input", timeout = SupersetConfigOptions::SupersetWebserverTimeout))] - MissingWebServerTimeoutInSupersetConfig, - #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::client::Error, @@ -229,56 +145,12 @@ pub enum Error { source: crate::operations::pdb::Error, }, - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, - }, - - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to get required Labels"))] GetRequiredLabels { source: stackable_operator::kvp::KeyValuePairError, }, - #[snafu(display("failed to add Superset config settings"))] - AddSupersetConfig { source: crate::config::Error }, - - #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] - AddLdapVolumesAndVolumeMounts { - source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, - }, - - #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] - AddTlsVolumesAndVolumeMounts { - source: stackable_operator::commons::tls_verification::TlsClientDetailsError, - }, - - #[snafu(display( - "failed to write to String (Vec to be precise) containing superset config" - ))] - WriteToConfigFileString { source: std::io::Error }, - - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, - }, - #[snafu(display("SupersetCluster object is invalid"))] InvalidSupersetCluster { source: error_boundary::InvalidObject, @@ -289,21 +161,35 @@ pub enum Error { source: stackable_operator::commons::opa::Error, }, - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, - }, - #[snafu(display("failed to apply group listener"))] ApplyGroupListener { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to configure listener"))] - ListenerConfiguration { source: crate::listener::Error }, + #[snafu(display("failed to build listener"))] + BuildListener { + source: crate::resources::listener::Error, + }, + + #[snafu(display("failed to build service"))] + BuildService { + source: crate::resources::service::Error, + }, - #[snafu(display("failed to configure service"))] - ServiceConfiguration { source: crate::service::Error }, + #[snafu(display("failed to build statefulset"))] + BuildStatefulSet { + source: crate::resources::statefulset::Error, + }, + + #[snafu(display("failed to build deployment"))] + BuildDeployment { + source: crate::resources::deployment::Error, + }, + + #[snafu(display("failed to build configmap"))] + BuildConfigMap { + source: crate::resources::configmap::Error, + }, #[snafu(display("failed to resolve product image"))] ResolveProductImage { @@ -346,7 +232,6 @@ pub async fn reconcile_superset( crate::built_info::PKG_VERSION, ) .context(ResolveProductImageSnafu)?; - let superset_role = SupersetRole::Node; let cluster_operation_cond_builder = ClusterOperationsConditionBuilder::new(&superset.spec.cluster_config.cluster_operation); @@ -358,34 +243,33 @@ pub async fn reconcile_superset( .await .context(InvalidAuthenticationConfigSnafu)?; - let validated_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &transform_all_roles_to_config( - superset, - &[( - superset_role.to_string(), + let mut roles = HashMap::new(); + + for role in SupersetRole::iter() { + if let Some(resolved_role) = superset.get_role(&role) { + roles.insert( + role.to_string(), ( vec![ PropertyNameKind::Env, PropertyNameKind::File(SUPERSET_CONFIG_FILENAME.into()), ], - superset.spec.nodes.clone().context(NoNodeRoleSnafu)?, + resolved_role.clone(), ), - )] - .into(), - ) - .context(GenerateProductConfigSnafu)?, + ); + } + } + + let role_config = transform_all_roles_to_config(superset, &roles); + let validated_role_config = validate_all_roles_and_groups_config( + &resolved_product_image.product_version, + &role_config.context(GenerateProductConfigSnafu)?, &ctx.product_config, false, false, ) .context(InvalidProductConfigSnafu)?; - let role_node_config = validated_config - .get(superset_role.to_string().as_str()) - .map(Cow::Borrowed) - .unwrap_or_default(); - let mut cluster_resources = ClusterResources::new( APP_NAME, OPERATOR_NAME, @@ -433,113 +317,176 @@ pub async fn reconcile_superset( .await .context(CreateSecretKeySecretSnafu)?; - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + let mut statefulset_cond_builder = StatefulSetConditionBuilder::default(); + let mut deployment_cond_builder = DeploymentConditionBuilder::default(); - for (rolegroup_name, rolegroup_config) in role_node_config.iter() { - let rolegroup = superset.node_rolegroup_ref(rolegroup_name); + for (role_name, role_config) in validated_role_config.iter() { + let superset_role = SupersetRole::from_str(role_name).context(ParseRoleSnafu)?; - let config = superset - .merged_config(&SupersetRole::Node, &rolegroup) - .context(FailedToResolveConfigSnafu)?; + for (rolegroup_name, rolegroup_config) in role_config.iter() { + let rolegroup = superset.rolegroup_ref(&superset_role, rolegroup_name); - let rg_configmap = build_rolegroup_config_map( - superset, - &resolved_product_image, - &rolegroup, - rolegroup_config, - &auth_config, - &superset_opa_config, - &config.logging, - )?; - let rg_statefulset = build_server_rolegroup_statefulset( - superset, - &resolved_product_image, - &superset_role, - &rolegroup, - rolegroup_config, - &auth_config, - &rbac_sa.name_any(), - &config, - )?; - - let rg_metrics_service = - build_node_rolegroup_metrics_service(superset, &resolved_product_image, &rolegroup) - .context(ServiceConfigurationSnafu)?; - - let rg_headless_service = - build_node_rolegroup_headless_service(superset, &resolved_product_image, &rolegroup) - .context(ServiceConfigurationSnafu)?; + let config = superset + .merged_config(&superset_role, &rolegroup) + .context(FailedToResolveConfigSnafu)?; - cluster_resources - .add(client, rg_metrics_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + let rg_configmap = build_rolegroup_config_map( + superset, + &resolved_product_image, + &rolegroup, + rolegroup_config, + &auth_config, + &superset_opa_config, + &config.logging, + ) + .context(BuildConfigMapSnafu)?; - cluster_resources - .add(client, rg_headless_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + let rg_metrics_service = + build_node_rolegroup_metrics_service(superset, &resolved_product_image, &rolegroup) + .context(BuildServiceSnafu)?; + + let rg_headless_service = build_node_rolegroup_headless_service( + superset, + &resolved_product_image, + &rolegroup, + ) + .context(BuildServiceSnafu)?; - cluster_resources - .add(client, rg_configmap) - .await - .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - })?; - - // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts - // to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - ss_cond_builder.add( cluster_resources - .add(client, rg_statefulset.clone()) + .add(client, rg_metrics_service) .await - .with_context(|_| ApplyRoleGroupStatefulSetSnafu { + .with_context(|_| ApplyRoleGroupServiceSnafu { rolegroup: rolegroup.clone(), - })?, - ); - } + })?; - if let Some(listener_class) = &superset_role.listener_class_name(superset) { - if let Some(listener_group_name) = superset.group_listener_name(&superset_role) { - let group_listener = build_group_listener( - superset, - build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.product_version, - &superset_role.to_string(), - "none", - ), - listener_class.to_string(), - listener_group_name, - ) - .context(ListenerConfigurationSnafu)?; cluster_resources - .add(client, group_listener) + .add(client, rg_headless_service) .await - .context(ApplyGroupListenerSnafu)?; - } - } + .with_context(|_| ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + })?; - let generic_role_config = superset.generic_role_config(&superset_role); - if let Some(GenericRoleConfig { - pod_disruption_budget: pdb, - }) = generic_role_config - { - add_pdbs( - &pdb, - superset, - &superset_role, - client, - &mut cluster_resources, - ) - .await - .context(FailedToCreatePdbSnafu)?; + cluster_resources + .add(client, rg_configmap) + .await + .with_context(|_| ApplyRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + })?; + + match superset_role { + SupersetRole::Node => { + let rg_statefulset = build_server_rolegroup_statefulset( + superset, + &resolved_product_image, + &superset_role, + &rolegroup, + rolegroup_config, + &auth_config, + &rbac_sa.name_any(), + &config, + ) + .context(BuildStatefulSetSnafu)?; + + // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts + // to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + statefulset_cond_builder.add( + cluster_resources + .add(client, rg_statefulset.clone()) + .await + .with_context(|_| ApplyRoleGroupStatefulSetSnafu { + rolegroup: rolegroup.clone(), + })?, + ); + } + SupersetRole::Worker => { + let rg_worker_deployment = build_worker_rolegroup_deployment( + superset, + &resolved_product_image, + &superset_role, + &rolegroup, + rolegroup_config, + &rbac_sa.name_any(), + &config, + ) + .context(BuildDeploymentSnafu)?; + + // Note: The Deployment needs to be applied after all ConfigMaps and Secrets it mounts + // to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + deployment_cond_builder.add( + cluster_resources + .add(client, rg_worker_deployment.clone()) + .await + .with_context(|_| ApplyRoleGroupDeploymentSnafu { + rolegroup: rolegroup.clone(), + })?, + ); + } + SupersetRole::Beat => { + let rg_beat_deployment = build_beat_rolegroup_deployment( + superset, + &resolved_product_image, + &superset_role, + &rolegroup, + rolegroup_config, + &rbac_sa.name_any(), + &config, + ) + .context(BuildDeploymentSnafu)?; + + // Note: The Deployment needs to be applied after all ConfigMaps and Secrets it mounts + // to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + deployment_cond_builder.add( + cluster_resources + .add(client, rg_beat_deployment.clone()) + .await + .with_context(|_| ApplyRoleGroupDeploymentSnafu { + rolegroup: rolegroup.clone(), + })?, + ); + } + } + + if let Some(listener_class) = &superset_role.listener_class_name(superset) { + if let Some(listener_group_name) = superset.group_listener_name(&superset_role) { + let group_listener = build_group_listener( + superset, + build_recommended_labels( + superset, + SUPERSET_CONTROLLER_NAME, + &resolved_product_image.product_version, + &superset_role.to_string(), + "none", + ), + listener_class.to_string(), + listener_group_name, + ) + .context(BuildListenerSnafu)?; + cluster_resources + .add(client, group_listener) + .await + .context(ApplyGroupListenerSnafu)?; + } + } + + let generic_role_config = superset.generic_role_config(&superset_role); + if let Some(GenericRoleConfig { + pod_disruption_budget: pdb, + }) = generic_role_config + { + add_pdbs( + &pdb, + superset, + &superset_role, + client, + &mut cluster_resources, + ) + .await + .context(FailedToCreatePdbSnafu)?; + } + } } cluster_resources @@ -550,7 +497,11 @@ pub async fn reconcile_superset( let status = SupersetClusterStatus { conditions: compute_conditions( superset, - &[&ss_cond_builder, &cluster_operation_cond_builder], + &[ + &statefulset_cond_builder, + &deployment_cond_builder, + &cluster_operation_cond_builder, + ], ), }; client @@ -561,538 +512,6 @@ pub async fn reconcile_superset( Ok(Action::await_change()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - authentication_config: &SupersetClientAuthenticationDetailsResolved, - superset_opa_config: &Option, - logging: &Logging, -) -> Result { - let mut config_properties = BTreeMap::new(); - let mut imports = PYTHON_IMPORTS.to_vec(); - // TODO: this is true per default for versions 3.0.0 and up. - // We deactivate it here to keep existing functionality. - // However this is a security issue and should be configured properly - // Issue: https://github.com/stackabletech/superset-operator/issues/416 - config_properties.insert("TALISMAN_ENABLED".to_string(), "False".to_string()); - - config::add_superset_config(&mut config_properties, authentication_config) - .context(AddSupersetConfigSnafu)?; - - // Adding opa configuration properties to config_properties. - // This will be injected as key/value pair in superset_config.py - if let Some(opa_config) = superset_opa_config { - // If opa role mapping is configured, insert CustomOpaSecurityManager import - imports.extend(OPA_IMPORTS); - - config_properties.extend(opa_config.as_config()); - } - - // The order here should be kept in order to preserve overrides. - // No properties should be added after this extend. - config_properties.extend( - rolegroup_config - .get(&PropertyNameKind::File( - SUPERSET_CONFIG_FILENAME.to_string(), - )) - .cloned() - .unwrap_or_default(), - ); - - let mut config_file = Vec::new(); - - // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) - if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; - } - let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - - flask_app_config_writer::write::( - &mut config_file, - config_properties.iter(), - &imports, - ) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; - - if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; - } - - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data( - SUPERSET_CONFIG_FILENAME, - String::from_utf8(config_file).unwrap(), - ); - - extend_config_map_with_log_config( - rolegroup, - logging, - &Container::Superset, - &Container::Vector, - &mut cm_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding -/// [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (via [`build_node_rolegroup_headless_service`] and metrics from [`build_node_rolegroup_metrics_service`]). -#[allow(clippy::too_many_arguments)] -fn build_server_rolegroup_statefulset( - superset: &SupersetCluster, - resolved_product_image: &ResolvedProductImage, - superset_role: &SupersetRole, - rolegroup_ref: &RoleGroupRef, - node_config: &HashMap>, - authentication_config: &SupersetClientAuthenticationDetailsResolved, - sa_name: &str, - merged_config: &SupersetConfig, -) -> Result { - let role = superset.get_role(superset_role).context(NoNodeRoleSnafu)?; - let role_group = role - .role_groups - .get(&rolegroup_ref.role_group) - .context(NoNodeRoleGroupSnafu)?; - - let recommended_object_labels = build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - superset, - SUPERSET_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(LabelBuildSnafu)?; - - let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&recommended_object_labels) - .context(MetadataBuildSnafu)? - .build(); - - let mut pb = &mut PodBuilder::new(); - - pb = pb - .metadata(metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .security_context( - PodSecurityContextBuilder::new() - .fs_group(1000) // Needed for secret-operator - .build(), - ) - .affinity(&merged_config.affinity) - .service_account_name(sa_name); - - // "METADATA" is the prefix for the env vars that hold the database credentials - // (e.g. METADATA_DATABASE_USERNAME, METADATA_DATABASE_PASSWORD). It should match - // the prefix used by the airflow-operator for consistency. - let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; - let metadata_database_connection_details = superset - .spec - .cluster_config - .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); - - let mut superset_cb = ContainerBuilder::new(&Container::Superset.to_string()) - .context(InvalidContainerNameSnafu)?; - - for (name, value) in node_config - .get(&PropertyNameKind::Env) - .cloned() - .unwrap_or_default() - { - if name == SupersetConfig::MAPBOX_SECRET_PROPERTY { - superset_cb.add_env_var_from_secret( - "MAPBOX_API_KEY", - &value, - "connections.mapboxApiKey", - ); - } else { - superset_cb.add_env_var(name, value); - }; - } - - // SECRET_KEY from auto-generated secret - superset_cb.add_env_var_from_secret( - "SECRET_KEY", - superset.shared_secret_key_secret_name(), - INTERNAL_SECRET_SECRET_KEY, - ); - - // Database connection URL from metadataDatabase - superset_cb.add_env_var( - "SQLALCHEMY_DATABASE_URI", - metadata_database_connection_details.url_template.clone(), - ); - - add_authentication_volumes_and_volume_mounts(authentication_config, &mut superset_cb, pb)?; - - let webserver_timeout = node_config - .get(&PropertyNameKind::File( - SUPERSET_CONFIG_FILENAME.to_string(), - )) - .context(MissingSupersetConfigInNodeConfigSnafu)? - .get(&SupersetConfigOptions::SupersetWebserverTimeout.to_string()) - .context(MissingWebServerTimeoutInSupersetConfigSnafu)?; - - let secret = &superset.spec.cluster_config.credentials_secret_name; - - superset_cb - .image_from_product_image(resolved_product_image) - .add_container_port("http", APP_PORT.into()) - .add_volume_mount(CONFIG_VOLUME_NAME, STACKABLE_CONFIG_DIR).context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR).context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR).context(AddVolumeMountSnafu)? - .add_env_var_from_secret("ADMIN_USERNAME", secret, "adminUser.username") - .add_env_var_from_secret("ADMIN_FIRSTNAME", secret, "adminUser.firstname") - .add_env_var_from_secret("ADMIN_LASTNAME", secret, "adminUser.lastname") - .add_env_var_from_secret("ADMIN_EMAIL", secret, "adminUser.email") - .add_env_var_from_secret("ADMIN_PASSWORD", secret, "adminUser.password") - // Needed by the `containerdebug` process to log it's tracing information to. - .add_env_var("CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug")) - .add_env_var("SSL_CERT_DIR", "/stackable/certs/") - .add_env_vars(authentication_env_vars(authentication_config)) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - - mkdir --parents {PYTHONPATH} - cp {STACKABLE_CONFIG_DIR}/* {PYTHONPATH} - cp {STACKABLE_LOG_CONFIG_DIR}/{LOG_CONFIG_FILE} {PYTHONPATH} - - {auth_commands} - - superset db upgrade - set +x - echo 'Running \"superset fab create-admin [...]\", which is not shown as it leaks the Superset admin credentials' - superset fab create-admin --username \"$ADMIN_USERNAME\" --firstname \"$ADMIN_FIRSTNAME\" --lastname \"$ADMIN_LASTNAME\" --email \"$ADMIN_EMAIL\" --password \"$ADMIN_PASSWORD\" - set -x - superset init - - {remove_vector_shutdown_file_command} - prepare_signal_handlers - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - gunicorn --bind 0.0.0.0:${{SUPERSET_PORT}} --worker-class gthread --threads 20 --timeout {webserver_timeout} --limit-request-line 0 --limit-request-field_size 0 'superset.app:create_app()' & - wait_for_termination $! - - {create_vector_shutdown_file_command} - ", - auth_commands = authentication_start_commands(authentication_config), - remove_vector_shutdown_file_command = - remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), - create_vector_shutdown_file_command = - create_vector_shutdown_file_command(STACKABLE_LOG_DIR), - }]) - .resources(merged_config.resources.clone().into()); - add_superset_container_probes(&mut superset_cb); - - // listener endpoints will use persistent volumes - // so that load balancers can hard-code the target addresses and - // that it is possible to connect to a consistent address - let pvcs = if let Some(group_listener_name) = superset.group_listener_name(superset_role) { - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(group_listener_name), - &unversioned_recommended_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_owned()) - .context(BuildListenerVolumeSnafu)?; - Some(vec![pvc]) - } else { - None - }; - - superset_cb - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; - - metadata_database_connection_details.add_to_container(&mut superset_cb); - - pb.add_container(superset_cb.build()); - add_graceful_shutdown_config(merged_config, pb).context(GracefulShutdownSnafu)?; - - let metrics_container = ContainerBuilder::new("metrics") - .context(InvalidContainerNameSnafu)? - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - - prepare_signal_handlers - /stackable/statsd_exporter & - wait_for_termination $! - "}]) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("64Mi") - .with_memory_limit("64Mi") - .build(), - ) - .build(); - - pb.add_volumes(controller_commons::create_volumes( - &rolegroup_ref.object_name(), - merged_config.logging.containers.get(&Container::Superset), - )) - .context(AddVolumeSnafu)?; - pb.add_container(metrics_container); - - if merged_config.logging.enable_vector_agent { - match &superset - .spec - .cluster_config - .vector_aggregator_config_map_name - { - Some(vector_aggregator_config_map_name) => { - pb.add_container( - product_logging::framework::vector_container( - resolved_product_image, - CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - merged_config.logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } - } - - let mut pod_template = pb.build_template(); - pod_template.merge_from(role.config.pod_overrides.clone()); - pod_template.merge_from(role_group.config.pod_overrides.clone()); - - Ok(StatefulSet { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(superset) - .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(superset, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&recommended_object_labels) - .context(MetadataBuildSnafu)? - .with_label( - Label::try_from(("restarter.stackable.tech/enabled", "true")) - .context(LabelBuildSnafu)?, - ) - .build(), - spec: Some(StatefulSetSpec { - // Set to `OrderedReady`, to make sure Pods start after another and the init commands don't run in parallel - pod_management_policy: Some("OrderedReady".to_string()), - replicas: role_group.replicas.map(i32::from), - selector: LabelSelector { - match_labels: Some( - Labels::role_group_selector( - superset, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), - ..LabelSelector::default() - }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), - template: pod_template, - volume_claim_templates: pvcs, - ..StatefulSetSpec::default() - }), - status: None, - }) -} - -fn add_superset_container_probes(superset_cb: &mut ContainerBuilder) { - let common = - ProbeBuilder::http_get_port_scheme_path(APP_PORT, None, Some("/health".to_owned())) - .with_period(Duration::from_secs(5)); - - superset_cb.startup_probe( - common - .clone() - .with_failure_threshold_duration(Duration::from_minutes_unchecked(10)) - .expect("const period is non-zero") - .build() - .expect("const duration does not overflow"), - ); - - // Remove it from the Service immediately - superset_cb.readiness_probe( - common - .clone() - .build() - .expect("const duration does not overflow"), - ); - // But only restart it after 3 failures - superset_cb.liveness_probe( - common - .with_failure_threshold(3) - .build() - .expect("const duration does not overflow"), - ); -} - -fn add_authentication_volumes_and_volume_mounts( - auth_config: &SupersetClientAuthenticationDetailsResolved, - cb: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // Different authentication entries can reference the same secret - // class or TLS certificate. It must be ensured that the volumes - // and volume mounts are only added once in such a case. - - let mut ldap_authentication_providers = BTreeSet::new(); - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Ldap { provider } => { - ldap_authentication_providers.insert(provider); - } - SupersetAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - } - } - } - - for provider in ldap_authentication_providers { - provider - .add_volumes_and_mounts(pb, vec![cb]) - .context(AddLdapVolumesAndVolumeMountsSnafu)?; - } - - for tls in tls_client_credentials { - tls.add_volumes_and_mounts(pb, vec![cb]) - .context(AddTlsVolumesAndVolumeMountsSnafu)?; - } - - Ok(()) -} - -fn authentication_env_vars( - auth_config: &SupersetClientAuthenticationDetailsResolved, -) -> Vec { - // Different OIDC authentication entries can reference the same - // client secret. It must be ensured that the env variables are only - // added once in such a case. - - let mut oidc_client_credentials_secrets = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Ldap { .. } => {} - SupersetAuthenticationClassResolved::Oidc { - client_auth_options: oidc, - .. - } => { - oidc_client_credentials_secrets - .insert(oidc.client_credentials_secret_ref.to_owned()); - } - } - } - - oidc_client_credentials_secrets - .iter() - .cloned() - .flat_map(oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts) - .collect() -} - -fn authentication_start_commands( - auth_config: &SupersetClientAuthenticationDetailsResolved, -) -> String { - let mut commands = Vec::new(); - - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - SupersetAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - - // WebPKI will be handled implicitly - } - SupersetAuthenticationClassResolved::Ldap { .. } => {} - } - } - - for tls in tls_client_credentials { - commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { - add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) - })); - } - - commands - .iter() - .flatten() - .cloned() - .collect::>() - .join("\n") -} - pub fn error_policy( _obj: Arc>, error: &Error, diff --git a/tests/templates/kuttl/celery-worker/00-limit-range.yaml b/tests/templates/kuttl/celery-worker/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 b/tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/celery-worker/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/celery-worker/10-assert.yaml b/tests/templates/kuttl/celery-worker/10-assert.yaml new file mode 100644 index 00000000..e9c60b15 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/10-install-postgresql.yaml b/tests/templates/kuttl/celery-worker/10-install-postgresql.yaml new file mode 100644 index 00000000..50c5ad67 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-postgresql + --namespace $NAMESPACE + --version 12.5.6 + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/celery-worker/20-assert.yaml b/tests/templates/kuttl/celery-worker/20-assert.yaml new file mode 100644 index 00000000..58d4fe84 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/20-assert.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-redis +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-redis-master +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-redis-replicas +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/20-install-redis.yaml b/tests/templates/kuttl/celery-worker/20-install-redis.yaml new file mode 100644 index 00000000..357f0fbc --- /dev/null +++ b/tests/templates/kuttl/celery-worker/20-install-redis.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-redis + --namespace $NAMESPACE + --version 17.11.3 + -f helm-bitnami-redis-values.yaml + --repo https://charts.bitnami.com/bitnami redis + --wait + timeout: 600 diff --git a/tests/templates/kuttl/celery-worker/30-assert.yaml.j2 b/tests/templates/kuttl/celery-worker/30-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/30-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/30-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/celery-worker/40-assert.yaml b/tests/templates/kuttl/celery-worker/40-assert.yaml new file mode 100644 index 00000000..f0618259 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/40-assert.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-superset +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-node-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-node +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-worker-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-worker +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-beat-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: superset-beat +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 new file mode 100644 index 00000000..6cb43f76 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/40-install-superset.yaml.j2 @@ -0,0 +1,95 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-superset +timeout: 300 +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-admin-credentials +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Superset + adminUser.lastname: Admin + adminUser.email: admin@superset.com + adminUser.password: admin +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-redis-credentials +stringData: + username: + password: superset +--- +apiVersion: superset.stackable.tech/v1alpha1 +kind: SupersetCluster +metadata: + name: superset +spec: + image: +{% if test_scenario['values']['superset'].find(",") > 0 %} + custom: "{{ test_scenario['values']['superset'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['superset'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['superset'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + credentialsSecretName: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials + celeryResultsBackend: + redis: + host: superset-redis-master + credentialsSecretName: superset-redis-credentials + celeryBroker: + redis: + host: superset-redis-master + credentialsSecretName: superset-redis-credentials +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + roleConfig: + listenerClass: external-unstable + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + superset_config.py: + FILE_FOOTER: |- + # Add endpoints that need to be exempt from CSRF protection + # Otherwise {"message": "400 Bad Request: The CSRF token is missing.", "error_type": "GENERIC_BACKEND_ERROR", "level": "error", "extra": {"issue_codes": [{"code": 1011, "message": "Issue 1011 - Superset encountered an unexpected error."}]}} + WTF_CSRF_ENABLED = False + roleGroups: + default: + replicas: 1 + workers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + beat: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/50-assert.yaml b/tests/templates/kuttl/celery-worker/50-assert.yaml new file mode 100644 index 00000000..f7f2887c --- /dev/null +++ b/tests/templates/kuttl/celery-worker/50-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-celery-worker +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/celery-worker/50-install-test-pod.yaml b/tests/templates/kuttl/celery-worker/50-install-test-pod.yaml new file mode 100644 index 00000000..3a64448f --- /dev/null +++ b/tests/templates/kuttl/celery-worker/50-install-test-pod.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-celery-worker + labels: + app: test-celery-worker +spec: + replicas: 1 + selector: + matchLabels: + app: test-celery-worker + template: + metadata: + labels: + app: test-celery-worker + spec: + containers: + - name: test-celery-worker + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + stdin: true + tty: true + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" diff --git a/tests/templates/kuttl/celery-worker/60-assert.yaml b/tests/templates/kuttl/celery-worker/60-assert.yaml new file mode 100644 index 00000000..8c58577c --- /dev/null +++ b/tests/templates/kuttl/celery-worker/60-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: copy-scripts +timeout: 300 +commands: + - script: kubectl cp -n $NAMESPACE ./create-database.sh test-celery-worker-0:/tmp + - script: kubectl cp -n $NAMESPACE ./run-select-query.sh test-celery-worker-0:/tmp diff --git a/tests/templates/kuttl/celery-worker/70-create-database.yaml b/tests/templates/kuttl/celery-worker/70-create-database.yaml new file mode 100644 index 00000000..bf902051 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/70-create-database.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl exec -n $NAMESPACE test-celery-worker-0 -- /tmp/create-database.sh diff --git a/tests/templates/kuttl/celery-worker/71-run-query.yaml b/tests/templates/kuttl/celery-worker/71-run-query.yaml new file mode 100644 index 00000000..cbd53d62 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/71-run-query.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl exec -n $NAMESPACE test-celery-worker-0 -- /tmp/run-select-query.sh diff --git a/tests/templates/kuttl/celery-worker/72-assert.yaml b/tests/templates/kuttl/celery-worker/72-assert.yaml new file mode 100644 index 00000000..05fb028e --- /dev/null +++ b/tests/templates/kuttl/celery-worker/72-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-redis +timeout: 300 +commands: + - script: kubectl exec superset-redis-master-0 --namespace $NAMESPACE -- redis-cli -h 127.0.0.1 -p 6379 --pass 'superset' KEYS '*' | grep 'superset_results' + - script: kubectl exec superset-redis-master-0 --namespace $NAMESPACE -- redis-cli -h 127.0.0.1 -p 6379 --pass 'superset' KEYS '*' | grep 'celery-task-meta-' diff --git a/tests/templates/kuttl/celery-worker/create-database.sh b/tests/templates/kuttl/celery-worker/create-database.sh new file mode 100755 index 00000000..965db149 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/create-database.sh @@ -0,0 +1,22 @@ +#!/bin/sh +ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" \ + -H "Content-Type: application/json" \ + -d '{ + "username": "admin", + "password": "admin", + "provider": "db" + }' | jq -r '.access_token') + +curl -X POST "http://superset-node:8088/api/v1/database/" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "database_name": "postgresql-integrationtest", + "sqlalchemy_uri": "postgresql+psycopg2://superset:superset@superset-postgresql:5432/superset", + "expose_in_sqllab": true, + "allow_run_async": true, + "allow_ctas": false, + "allow_cvas": false, + "allow_dml": false, + "extra": "{\"allows_virtual_table_explore\": true}" + }' diff --git a/tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..2e851682 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,44 @@ +--- +global: + security: + allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images + +image: + repository: bitnamilegacy/postgresql + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + securityContext: + runAsUser: auto + +metrics: + image: + repository: bitnamilegacy/postgres-exporter + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: superset + password: superset + database: superset diff --git a/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 b/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 new file mode 100644 index 00000000..28418f06 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/helm-bitnami-redis-values.yaml.j2 @@ -0,0 +1,65 @@ +--- +global: + commonLabels: + stackable.tech/vendor: Stackable + security: + allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images +image: + repository: bitnamilegacy/redis +sentinel: + image: + repository: bitnamilegacy/redis-sentinel +metrics: + image: + repository: bitnamilegacy/redis-exporter +kubectl: + image: + repository: bitnamilegacy/kubectl +sysctl: + image: + repository: bitnamilegacy/os-shell + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + containerSecurityContext: + runAsUser: auto + +master: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "200m" + limits: + memory: "128Mi" + cpu: "800m" + +replica: + replicaCount: 1 + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" + +auth: + password: superset diff --git a/tests/templates/kuttl/celery-worker/run-select-query.sh b/tests/templates/kuttl/celery-worker/run-select-query.sh new file mode 100755 index 00000000..97deda21 --- /dev/null +++ b/tests/templates/kuttl/celery-worker/run-select-query.sh @@ -0,0 +1,52 @@ +#!/bin/sh +ACCESS_TOKEN=$(curl -s -X POST "http://superset-node:8088/api/v1/security/login" \ + -H "Content-Type: application/json" \ + -d '{ + "username": "admin", + "password": "admin", + "provider": "db" + }' | jq -r '.access_token') + +EXECUTE_QUERY_RESPONSE=$(curl -X POST "http://superset-node:8088/api/v1/sqllab/execute/" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "database_id": 1, + "runAsync": true, + "sql": "SELECT username, first_name, last_name from public.ab_user;" + }') + +QUERY_ID=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.queryId') +QUERY_STATE=$(echo "$EXECUTE_QUERY_RESPONSE" | jq -r '.query.state') + +echo "Query started with ID: '$QUERY_ID' in state '$QUERY_STATE' ..." + +while [ "$QUERY_STATE" = "pending" ] || [ "$QUERY_STATE" = "running" ]; do + POLL_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/query/$QUERY_ID" \ + -H "Authorization: Bearer $ACCESS_TOKEN") + + QUERY_STATE=$(echo "$POLL_RESPONSE" | jq -r '.result.status') + echo "Current State: '$QUERY_STATE'" + + if [ "$QUERY_STATE" = "failed" ]; then + echo "Query failed!" + echo "$POLL_RESPONSE" + exit 1 + fi + + sleep 1 +done + +if [ "$QUERY_STATE" = "success" ]; then + RESULTS_KEY=$(echo "$POLL_RESPONSE" | jq -r '.result.results_key') + + echo "Query successful! Fetching data for results_key '$RESULTS_KEY' ..." + + DATA_RESPONSE=$(curl -s -X GET "http://superset-node:8088/api/v1/sqllab/results/?q=%7B%0A%20%20%22key%22%3A%20%22${RESULTS_KEY}%22%0A%7D" \ + -H "Authorization: Bearer $ACCESS_TOKEN") + + echo "$DATA_RESPONSE" | jq '.data' +else + echo "Query finished with unexpected state: $QUERY_STATE" + exit 1 +fi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 2b4d9d83..da87a2cd 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -64,6 +64,10 @@ tests: dimensions: - superset - openshift + - name: celery-worker + dimensions: + - superset + - openshift suites: - name: nightly patch: