From e647c1395a4d2cc7280054a38fd562f200132c04 Mon Sep 17 00:00:00 2001 From: Miretpl Date: Fri, 10 Apr 2026 22:53:23 +0200 Subject: [PATCH 1/3] Add workers.celery.logGroomerSidecar section --- .../docs/setting-resources-for-containers.rst | 2 +- chart/templates/NOTES.txt | 96 ++++++ chart/values.schema.json | 307 +++++++++++++++++- chart/values.yaml | 62 ++++ helm-tests/tests/chart_utils/log_groomer.py | 151 ++++++--- .../airflow_aux/test_container_lifecycle.py | 71 +++- .../helm_tests/airflow_core/test_worker.py | 7 + .../security/test_security_context.py | 25 +- 8 files changed, 655 insertions(+), 66 deletions(-) diff --git a/chart/docs/setting-resources-for-containers.rst b/chart/docs/setting-resources-for-containers.rst index 600b690778ddd..1402d465d79cf 100644 --- a/chart/docs/setting-resources-for-containers.rst +++ b/chart/docs/setting-resources-for-containers.rst @@ -31,7 +31,7 @@ Possible containers where resources can be configured include: * Main Airflow containers and their sidecars. You can add the resources for these containers through the following parameters: * ``workers.resources`` - * ``workers.logGroomerSidecar.resources`` + * ``workers.celery.logGroomerSidecar.resources`` * ``workers.kerberosSidecar.resources`` * ``workers.kerberosInitContainer.resources`` * ``scheduler.resources`` diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index baee62cb88138..205fd39ec1930 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -829,6 +829,102 @@ DEPRECATION WARNING: {{- end }} +{{- if not .Values.workers.logGroomerSidecar.enabled }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.enabled` has been renamed to `workers.celery.logGroomerSidecar.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.command) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.command` has been renamed to `workers.celery.logGroomerSidecar.command`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (.Values.workers.logGroomerSidecar.args | toJson) (list "bash" "/clean-logs" | toJson) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.args` has been renamed to `workers.celery.logGroomerSidecar.args`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.retentionDays) 15 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.retentionDays` has been renamed to `workers.celery.logGroomerSidecar.retentionDays`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.retentionMinutes) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.retentionMinutes` has been renamed to `workers.celery.logGroomerSidecar.retentionMinutes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.frequencyMinutes) 15 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.frequencyMinutes` has been renamed to `workers.celery.logGroomerSidecar.frequencyMinutes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.maxSizeBytes) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.maxSizeBytes` has been renamed to `workers.celery.logGroomerSidecar.maxSizeBytes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.maxSizePercent) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.maxSizePercent` has been renamed to `workers.celery.logGroomerSidecar.maxSizePercent`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.resources) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.resources` has been renamed to `workers.celery.logGroomerSidecar.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.securityContexts.container) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.securityContexts.container` has been renamed to `workers.celery.logGroomerSidecar.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.env) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.env` has been renamed to `workers.celery.logGroomerSidecar.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.containerLifecycleHooks) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.containerLifecycleHooks` has been renamed to `workers.celery.logGroomerSidecar.containerLifecycleHooks`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not .Values.workers.waitForMigrations.enabled }} DEPRECATION WARNING: diff --git a/chart/values.schema.json b/chart/values.schema.json index 8d152146ca1c7..eed378e172377 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2436,8 +2436,150 @@ } }, "logGroomerSidecar": { - "$ref": "#/definitions/logGroomerConfigType", - "description": "Configuration for Airflow Celery worker log groomer sidecar" + "description": "Configuration for Airflow Celery worker log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.enabled`` instead).", + "type": "boolean", + "default": true + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.command`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.args`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "/clean-logs" + ] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention time is retentionDays + retentionMinutes.", + "type": "integer", + "default": 15 + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention time is retentionDays + retentionMinutes.", + "type": "integer", + "default": 0 + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.frequencyMinutes`` instead).", + "type": "integer", + "default": 15 + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes (deprecated, use ``workers.celery.logGroomerSidecar.maxSizeBytes`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": "integer", + "default": 0, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space (deprecated, use ``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated) (deprecated, use ``workers.celery.logGroomerSidecar.env`` instead).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.resources`` instead).", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } }, "securityContext": { "description": "Security context for the Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.securityContexts`` and/or ``workers.kubernetes.securityContexts`` instead). If not set, the values from `securityContext` will be used.", @@ -3506,6 +3648,167 @@ "type": "string" } }, + "logGroomerSidecar": { + "description": "Configuration for Airflow Celery worker log groomer sidecar.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": [ + "integer", + "null" + ], + "default": null, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "type": [ + "integer", + "null" + ], + "default": null, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar. If not set, the values from ``workers.containerLifecycleHooks`` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar. If not set, the values from ``workers.securityContexts`` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, "waitForMigrations": { "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 8a3290df5aaec..0445e19baea03 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1142,33 +1142,43 @@ workers: labels: {} # Log groomer configuration for Airflow Celery workers + # (deprecated, use `workers.celery.logGroomerSidecar` instead) logGroomerSidecar: # Whether to deploy the Airflow Celery worker log groomer sidecar + # (deprecated, use `workers.celery.logGroomerSidecar.enabled` instead) enabled: true # Command to use when running the Airflow Celery worker log groomer sidecar (templated) + # (deprecated, use `workers.celery.logGroomerSidecar.command` instead) command: ~ # Args to use when running the Airflow Celery worker log groomer sidecar (templated) + # (deprecated, use `workers.celery.logGroomerSidecar.args` instead) args: ["bash", "/clean-logs"] # Number of days to retain logs + # (deprecated, use `workers.celery.logGroomerSidecar.retentionDays` instead) retentionDays: 15 # Number of minutes to retain logs. # This can be used for finer granularity than days. # Total retention is retentionDays + retentionMinutes. + # (deprecated, use `workers.celery.logGroomerSidecar.retentionMinutes` instead) retentionMinutes: 0 # Frequency to attempt to groom logs (in minutes) + # (deprecated, use `workers.celery.logGroomerSidecar.frequencyMinutes` instead) frequencyMinutes: 15 # Max size of logs in bytes. 0 = disabled + # (deprecated, use `workers.celery.logGroomerSidecar.maxSizeBytes` instead) maxSizeBytes: 0 # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + # (deprecated, use `workers.celery.logGroomerSidecar.maxSizePercent` instead) maxSizePercent: 0 + # (deprecated, use `workers.celery.logGroomerSidecar.resources` instead) resources: {} # limits: # cpu: 100m @@ -1178,11 +1188,18 @@ workers: # memory: 128Mi # Detailed default security context for logGroomerSidecar for container level + # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts` instead) securityContexts: + # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts.container` instead) container: {} + # (deprecated, use `workers.celery.logGroomerSidecar.env` instead) env: [] + # Container level lifecycle hooks + # (deprecated, use `workers.celery.logGroomerSidecar.containerLifecycleHooks` instead) + containerLifecycleHooks: {} + # Configuration of wait-for-airflow-migration init container for Airflow Celery workers # (deprecated, use `workers.celery.waitForMigrations` instead) waitForMigrations: @@ -1533,6 +1550,51 @@ workers: # Pod annotations for the Airflow Celery workers (templated) podAnnotations: {} + # Log groomer configuration for Airflow Celery workers + logGroomerSidecar: + # Whether to deploy the Airflow Celery worker log groomer sidecar + enabled: ~ + + # Command to use when running the Airflow Celery worker log groomer sidecar (templated) + command: ~ + + # Args to use when running the Airflow Celery worker log groomer sidecar (templated) + args: [] + + # Number of days to retain logs + retentionDays: ~ + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is retentionDays + retentionMinutes. + retentionMinutes: ~ + + # Frequency to attempt to groom logs (in minutes) + frequencyMinutes: ~ + + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: ~ + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + maxSizePercent: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for logGroomerSidecar for container level + securityContexts: + container: {} + + env: [] + + # Container level lifecycle hooks + containerLifecycleHooks: {} + # Configuration of wait-for-airflow-migration init container for Airflow Celery workers waitForMigrations: # Whether to create init container to wait for db migrations diff --git a/helm-tests/tests/chart_utils/log_groomer.py b/helm-tests/tests/chart_utils/log_groomer.py index 4d88c7da9472e..308a03ffa6b2c 100644 --- a/helm-tests/tests/chart_utils/log_groomer.py +++ b/helm-tests/tests/chart_utils/log_groomer.py @@ -26,18 +26,27 @@ class LogGroomerTestBase: obj_name: str = "" folder: str = "" + def get_show_only(self): + if self.obj_name == "workers-celery": + return [f"templates/{self.folder}/worker-deployment.yaml"] + + return [f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] + def test_log_groomer_collector_default_enabled(self): if self.obj_name == "dag-processor": values = {"dagProcessor": {"enabled": True}} else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + if self.obj_name == "workers-celery": + container_name = "worker-log-groomer" + else: + container_name = f"{self.obj_name}-log-groomer" + + docs = render_chart(values=values, show_only=self.get_show_only()) assert len(jmespath.search("spec.template.spec.containers", docs[0])) == 2 - assert f"{self.obj_name}-log-groomer" in [ + assert container_name in [ c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0]) ] @@ -49,14 +58,18 @@ def test_log_groomer_collector_can_be_disabled(self): "logGroomerSidecar": {"enabled": False}, } } + elif self.obj_name == "workers-celery": + values = { + "workers": { + "celery": { + "logGroomerSidecar": {"enabled": False}, + } + } + } else: values = {f"{self.folder}": {"logGroomerSidecar": {"enabled": False}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) - + docs = render_chart(values=values, show_only=self.get_show_only()) actual = jmespath.search("spec.template.spec.containers", docs[0]) assert len(actual) == 1 @@ -67,9 +80,7 @@ def test_log_groomer_collector_default_command_and_args(self): else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == ["bash", "/clean-logs"] @@ -80,13 +91,13 @@ def test_log_groomer_collector_default_retention_days(self): else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) - assert {"name": "AIRFLOW__LOG_RETENTION_DAYS", "value": "15"} in jmespath.search( - "spec.template.spec.containers[1].env", docs[0] + assert ( + jmespath.search("spec.template.spec.containers[1].env[0].name", docs[0]) + == "AIRFLOW__LOG_RETENTION_DAYS" ) + assert jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0]) == "15" def test_log_groomer_collector_custom_env(self): env = [ @@ -96,6 +107,8 @@ def test_log_groomer_collector_custom_env(self): if self.obj_name == "dag-processor": values = {"dagProcessor": {"enabled": True, "logGroomerSidecar": {"env": env}}} + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"env": env}}}} else: values = { "workers": {"logGroomerSidecar": {"env": env}}, @@ -103,9 +116,7 @@ def test_log_groomer_collector_custom_env(self): "triggerer": {"logGroomerSidecar": {"env": env}}, } - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert {"name": "APP_RELEASE_NAME", "value": "release-name-airflow"} in jmespath.search( "spec.template.spec.containers[1].env", docs[0] @@ -124,16 +135,22 @@ def test_log_groomer_command_and_args_overrides(self, command, args): "logGroomerSidecar": {"command": command, "args": args}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"command": command, "args": args}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"command": command, "args": args}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert command == jmespath.search("spec.template.spec.containers[1].command", docs[0]) - assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0]) + + if self.obj_name == "workers-celery" and args is None: + assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == [ + "bash", + "/clean-logs", + ] + else: + assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0]) def test_log_groomer_command_and_args_overrides_are_templated(self): if self.obj_name == "dag-processor": @@ -146,6 +163,17 @@ def test_log_groomer_command_and_args_overrides_are_templated(self): }, } } + elif self.obj_name == "workers-celery": + values = { + "workers": { + "celery": { + "logGroomerSidecar": { + "command": ["{{ .Release.Name }}"], + "args": ["{{ .Release.Service }}"], + } + } + } + } else: values = { f"{self.folder}": { @@ -156,10 +184,7 @@ def test_log_groomer_command_and_args_overrides_are_templated(self): } } - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) == ["release-name"] assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == ["Helm"] @@ -170,13 +195,12 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re values = { "dagProcessor": {"enabled": True, "logGroomerSidecar": {"retentionDays": retention_days}} } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"retentionDays": retention_days}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"retentionDays": retention_days}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if retention_result: assert ( @@ -186,6 +210,15 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re ) == retention_result ) + elif self.obj_name == "workers-celery" and retention_result is None: + # Testing backward compatibility of move from workers to workers.celery + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value | [0]", + docs[0], + ) + == "15" + ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2 @@ -198,13 +231,12 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque "logGroomerSidecar": {"frequencyMinutes": frequency_minutes}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"frequencyMinutes": frequency_minutes}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"frequencyMinutes": frequency_minutes}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if frequency_result: assert ( @@ -214,6 +246,15 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque ) == frequency_result ) + elif self.obj_name == "workers-celery" and frequency_result is None: + # Testing backward compatibility of move from workers to workers.celery + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value | [0]", + docs[0], + ) + == "15" + ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2 @@ -228,13 +269,12 @@ def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes, max_size_res "logGroomerSidecar": {"maxSizeBytes": max_size_bytes}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if max_size_result: assert ( @@ -262,13 +302,12 @@ def test_log_groomer_max_size_percent_overrides(self, max_size_percent, max_size "logGroomerSidecar": {"maxSizePercent": max_size_percent}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if max_size_result: assert ( @@ -300,6 +339,19 @@ def test_log_groomer_resources(self): }, } } + elif self.obj_name == "workers-celery": + values = { + "workers": { + "celery": { + "logGroomerSidecar": { + "resources": { + "requests": {"memory": "2Gi", "cpu": "1"}, + "limits": {"memory": "3Gi", "cpu": "2"}, + } + } + } + } + } else: values = { f"{self.folder}": { @@ -312,10 +364,7 @@ def test_log_groomer_resources(self): } } - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert jmespath.search("spec.template.spec.containers[1].resources", docs[0]) == { "limits": { @@ -334,9 +383,7 @@ def test_log_groomer_has_airflow_home(self): else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert ( jmespath.search("spec.template.spec.containers[1].env[?name=='AIRFLOW_HOME'].name | [0]", docs[0]) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py index 04bef2de7dfef..f701e3f9ba8aa 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py @@ -265,25 +265,78 @@ def test_worker_kerberos_container_setting(self, workers_values, expected_hook_t # Test container lifecycle hooks for log-groomer-sidecar main container @pytest.mark.parametrize("hook_type", ["preStop", "postStart"]) - def test_log_groomer_sidecar_container_setting(self, hook_type): + def test_log_groomer_sidecar_container_setting_scheduler(self, hook_type): docs = render_chart( name=RELEASE_NAME, values={ "scheduler": { "logGroomerSidecar": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}} }, - "workers": { - "logGroomerSidecar": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}} - }, }, show_only=[ "templates/scheduler/scheduler-deployment.yaml", + ], + ) + + assert ( + jmespath.search(f"spec.template.spec.containers[1].lifecycle.{hook_type}", docs[0]) + == LIFECYCLE_PARSED + ) + + @pytest.mark.parametrize( + ("workers_values", "expected"), + [ + ( + {"logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}}, + {"preStop": LIFECYCLE_PARSED}, + ), + ( + {"logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}}, + {"postStart": LIFECYCLE_PARSED}, + ), + ( + { + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}} + } + }, + {"preStop": LIFECYCLE_PARSED}, + ), + ( + { + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}} + } + }, + {"postStart": LIFECYCLE_PARSED}, + ), + ( + { + "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}, + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}} + }, + }, + {"preStop": LIFECYCLE_PARSED}, + ), + ( + { + "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}, + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}} + }, + }, + {"postStart": LIFECYCLE_PARSED}, + ), + ], + ) + def test_log_groomer_sidecar_container_setting(self, workers_values, expected): + docs = render_chart( + name=RELEASE_NAME, + values={"workers": workers_values}, + show_only=[ "templates/workers/worker-deployment.yaml", ], ) - for doc in docs: - assert ( - jmespath.search(f"spec.template.spec.containers[1].lifecycle.{hook_type}", doc) - == LIFECYCLE_PARSED - ) + assert jmespath.search("spec.template.spec.containers[1].lifecycle", docs[0]) == expected diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index 38bba315458d2..376a2d30c08fb 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -2290,6 +2290,13 @@ class TestWorkerLogGroomer(LogGroomerTestBase): folder = "workers" +class TestWorkerCeleryLogGroomer(LogGroomerTestBase): + """Worker Celery groomer.""" + + obj_name = "workers-celery" + folder = "workers" + + class TestWorkerKedaAutoScaler: """Tests worker keda auto scaler.""" diff --git a/helm-tests/tests/helm_tests/security/test_security_context.py b/helm-tests/tests/helm_tests/security/test_security_context.py index a5e9d70816b5f..39e9281d04b40 100644 --- a/helm-tests/tests/helm_tests/security/test_security_context.py +++ b/helm-tests/tests/helm_tests/security/test_security_context.py @@ -554,13 +554,34 @@ def test_main_container_setting(self, workers_values): assert ctx_value == jmespath.search("spec.template.spec.containers[0].securityContext", doc) # Test securityContexts for log-groomer-sidecar main container - def test_log_groomer_sidecar_container_setting(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"logGroomerSidecar": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, + { + "celery": { + "logGroomerSidecar": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + } + }, + { + "logGroomerSidecar": {"securityContexts": {"container": {"runAsUser": 20}}}, + "celery": { + "logGroomerSidecar": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + }, + }, + ], + ) + def test_log_groomer_sidecar_container_setting(self, workers_values): ctx_value = {"allowPrivilegeEscalation": False} spec = {"logGroomerSidecar": {"securityContexts": {"container": ctx_value}}} docs = render_chart( values={ "scheduler": spec, - "workers": spec, + "workers": workers_values, "dagProcessor": spec, "triggerer": spec, }, From ed109ddb8f21d4db757fe5ea30d8929730b5fe16 Mon Sep 17 00:00:00 2001 From: Miretpl Date: Fri, 10 Apr 2026 22:55:29 +0200 Subject: [PATCH 2/3] Add newsfragment --- chart/newsfragments/65033.significant.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 chart/newsfragments/65033.significant.rst diff --git a/chart/newsfragments/65033.significant.rst b/chart/newsfragments/65033.significant.rst new file mode 100644 index 0000000000000..ee4ec2bfd2092 --- /dev/null +++ b/chart/newsfragments/65033.significant.rst @@ -0,0 +1 @@ +``workers.logGroomerSidecar`` section is now deprecated in favor of ``workers.celery.logGroomerSidecar``. Please update your configuration accordingly. From f6d39c38312af738d9b278429e55ea0d3f358fdc Mon Sep 17 00:00:00 2001 From: Miretpl Date: Sat, 11 Apr 2026 16:31:12 +0200 Subject: [PATCH 3/3] Fix spellcheck & tests --- chart/values.schema.json | 18 +++++++++--------- chart/values.yaml | 18 +++++++++--------- helm-tests/tests/chart_utils/log_groomer.py | 6 ++---- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index eed378e172377..984149f26b241 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2471,12 +2471,12 @@ ] }, "retentionDays": { - "description": "Number of days to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention time is retentionDays + retentionMinutes.", + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": "integer", "default": 15 }, "retentionMinutes": { - "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention time is retentionDays + retentionMinutes.", + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": "integer", "default": 0 }, @@ -2492,7 +2492,7 @@ "minimum": 0 }, "maxSizePercent": { - "description": "Max size of logs as a percentage of total disk space (deprecated, use ``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "description": "Max size of logs as a percentage of total disk space (deprecated, use ``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", "type": "integer", "default": 0, "minimum": 0, @@ -3684,7 +3684,7 @@ "default": [] }, "retentionDays": { - "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": [ "integer", "null" @@ -3692,7 +3692,7 @@ "default": null }, "retentionMinutes": { - "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": [ "integer", "null" @@ -3717,7 +3717,7 @@ "minimum": 0 }, "maxSizePercent": { - "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", "type": [ "integer", "null" @@ -15001,12 +15001,12 @@ ] }, "retentionDays": { - "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": "integer", "default": 15 }, "retentionMinutes": { - "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": "integer", "default": 0 }, @@ -15022,7 +15022,7 @@ "minimum": 0 }, "maxSizePercent": { - "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", "type": "integer", "default": 0, "minimum": 0, diff --git a/chart/values.yaml b/chart/values.yaml index 0445e19baea03..fa56e57bebc0b 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1162,7 +1162,7 @@ workers: # Number of minutes to retain logs. # This can be used for finer granularity than days. - # Total retention is retentionDays + retentionMinutes. + # Total retention is `retentionDays` + `retentionMinutes`. # (deprecated, use `workers.celery.logGroomerSidecar.retentionMinutes` instead) retentionMinutes: 0 @@ -1174,7 +1174,7 @@ workers: # (deprecated, use `workers.celery.logGroomerSidecar.maxSizeBytes` instead) maxSizeBytes: 0 - # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. # (deprecated, use `workers.celery.logGroomerSidecar.maxSizePercent` instead) maxSizePercent: 0 @@ -1187,7 +1187,7 @@ workers: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts` instead) securityContexts: # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts.container` instead) @@ -1566,7 +1566,7 @@ workers: # Number of minutes to retain logs. # This can be used for finer granularity than days. - # Total retention is retentionDays + retentionMinutes. + # Total retention is `retentionDays` + `retentionMinutes`. retentionMinutes: ~ # Frequency to attempt to groom logs (in minutes) @@ -1575,7 +1575,7 @@ workers: # Max size of logs in bytes. 0 = disabled maxSizeBytes: ~ - # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. maxSizePercent: ~ resources: {} @@ -1586,7 +1586,7 @@ workers: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level securityContexts: container: {} @@ -1973,7 +1973,7 @@ scheduler: # Max size of logs in bytes. 0 = disabled maxSizeBytes: 0 - # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. maxSizePercent: 0 resources: {} @@ -1984,7 +1984,7 @@ scheduler: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level securityContexts: container: {} @@ -2907,7 +2907,7 @@ triggerer: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level securityContexts: container: {} diff --git a/helm-tests/tests/chart_utils/log_groomer.py b/helm-tests/tests/chart_utils/log_groomer.py index 308a03ffa6b2c..36a2c1eb7ae2b 100644 --- a/helm-tests/tests/chart_utils/log_groomer.py +++ b/helm-tests/tests/chart_utils/log_groomer.py @@ -93,11 +93,9 @@ def test_log_groomer_collector_default_retention_days(self): docs = render_chart(values=values, show_only=self.get_show_only()) - assert ( - jmespath.search("spec.template.spec.containers[1].env[0].name", docs[0]) - == "AIRFLOW__LOG_RETENTION_DAYS" + assert {"name": "AIRFLOW__LOG_RETENTION_DAYS", "value": "15"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] ) - assert jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0]) == "15" def test_log_groomer_collector_custom_env(self): env = [