diff --git a/chart/docs/customizing-labels.rst b/chart/docs/customizing-labels.rst index 978dbec2152ca..80b56cee80924 100644 --- a/chart/docs/customizing-labels.rst +++ b/chart/docs/customizing-labels.rst @@ -42,7 +42,7 @@ You can also set specific labels for individual Airflow components, which will b If the same label key exists in both global and component-specific labels, the component-specific value takes precedence (overrides the global value). This allows you to customize labels for specific components while still maintaining common global labels across all resources. -For example, to add specific labels to different components: +For example, to add specific labels to different components like scheduler or api-server: .. code-block:: yaml :caption: values.yaml @@ -56,11 +56,6 @@ For example, to add specific labels to different components: labels: role: scheduler - # Worker specific labels - workers: - labels: - role: worker - # API Server specific labels apiServer: labels: diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 305e7b5e7235e..6449fe329ad88 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -39,8 +39,8 @@ metadata: tier: airflow component: worker release: {{ .Release.Name }} - {{- if or (.Values.labels) (.Values.workers.labels) }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} + {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} {{- end }} annotations: {{- tpl (toYaml $podAnnotations) . | nindent 4 }} diff --git a/chart/newsfragments/65030.significant.rst b/chart/newsfragments/65030.significant.rst new file mode 100644 index 0000000000000..055bfa7651919 --- /dev/null +++ b/chart/newsfragments/65030.significant.rst @@ -0,0 +1 @@ +``workers.labels`` field is now deprecated in favor of ``workers.celery.labels`` and ``workers.kubernetes.labels``. Please update your configuration accordingly. diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 205fd39ec1930..b5038eaa55fd4 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -813,6 +813,14 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.labels) }} + + DEPRECATION WARNING: + `workers.labels` has been renamed to `workers.celery.labels`/`workers.kubernetes.labels`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.workers.volumeClaimTemplates) }} DEPRECATION WARNING: diff --git a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml index 7530f0732d3be..b74474b42f288 100644 --- a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml +++ b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml @@ -32,8 +32,8 @@ metadata: release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} + {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with (.Values.workers.kubernetes.serviceAccount.annotations | default .Values.workers.serviceAccount.annotations) }} annotations: {{- toYaml . | nindent 4 }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 984149f26b241..a9560366ffe5a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2428,7 +2428,7 @@ "x-docsSection": "Common" }, "labels": { - "description": "Labels to add to the Airflow Celery workers objects and pods created with pod-template-file.", + "description": "Labels to add to the Airflow Celery workers objects and pods created with pod-template-file (deprecated, use ``workers.celery.labels`` and/or ``workers.kubernetes.labels`` instead).", "type": "object", "default": {}, "additionalProperties": { @@ -3648,6 +3648,14 @@ "type": "string" } }, + "labels": { + "description": "Labels to add to the Airflow Celery workers objects.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, "logGroomerSidecar": { "description": "Configuration for Airflow Celery worker log groomer sidecar.", "type": "object", @@ -4349,6 +4357,14 @@ "type": "string" } }, + "labels": { + "description": "Labels to add to the pods created with pod-template-file.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, "schedulerName": { "description": "Specify kube scheduler name for pods created with pod-template-file.", "type": [ diff --git a/chart/values.yaml b/chart/values.yaml index fa56e57bebc0b..75e0c979656cd 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1139,6 +1139,7 @@ workers: podAnnotations: {} # Labels specific to Airflow Celery workers objects and pods created with pod-template-file + # (deprecated, use `workers.celery.labels` and/or `workers.kubernetes.labels` instead) labels: {} # Log groomer configuration for Airflow Celery workers @@ -1550,6 +1551,9 @@ workers: # Pod annotations for the Airflow Celery workers (templated) podAnnotations: {} + # Labels specific to Airflow Celery workers objects + labels: {} + # Log groomer configuration for Airflow Celery workers logGroomerSidecar: # Whether to deploy the Airflow Celery worker log groomer sidecar @@ -1778,6 +1782,9 @@ workers: # Pod annotations for the pods created with pod-template-file (templated) podAnnotations: {} + # Labels specific to pods created with pod-template-file + labels: {} + schedulerName: ~ # Airflow scheduler settings diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index ed57a426ce514..55a1fb94d4c4b 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -1372,20 +1372,38 @@ def test_should_add_extraEnvs(self): "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, } in jmespath.search("spec.containers[0].env", docs[0]) - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "kubernetes": { + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "kubernetes": { + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels @pytest.mark.parametrize( "workers_values", diff --git a/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py index b3ba14679108d..2e18abad536df 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py @@ -57,22 +57,35 @@ def test_pod_disruption_budget_name(self, workers_values): @pytest.mark.parametrize( "workers_values", [ - {"podDisruptionBudget": {"enabled": True}}, - {"celery": {"podDisruptionBudget": {"enabled": True}}}, + {"podDisruptionBudget": {"enabled": True}, "labels": {"test_label": "test_label_value"}}, + { + "celery": {"podDisruptionBudget": {"enabled": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "podDisruptionBudget": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "podDisruptionBudget": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, ], ) def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( - values={ - "workers": { - **workers_values, - "labels": {"test_label": "test_label_value"}, - }, - }, + values={"workers": workers_values}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels @pytest.mark.parametrize( "workers_values", diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index 376a2d30c08fb..10124a2500873 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -471,19 +471,37 @@ def test_should_add_extraEnvs_to_wait_for_migration_container(self, workers_valu "spec.template.spec.initContainers[0].env", docs[0] ) - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert "test_label" in jmespath.search("spec.template.metadata.labels", docs[0]) - assert jmespath.search("spec.template.metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("spec.template.metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels @pytest.mark.parametrize( "workers_values", @@ -2300,20 +2318,40 @@ class TestWorkerCeleryLogGroomer(LogGroomerTestBase): class TestWorkerKedaAutoScaler: """Tests worker keda auto scaler.""" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": {"keda": {"enabled": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "keda": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "keda": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "celery": {"keda": {"enabled": True}}, - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels def test_should_remove_replicas_field(self): docs = render_chart( @@ -2469,20 +2507,37 @@ def test_should_be_disabled_on_keda_enabled(self, workers_values): assert len(docs) == 1 - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"celery": {"hpa": {"enabled": True}}, "labels": {"test_label": "test_label_value"}}, + { + "celery": { + "hpa": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "hpa": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "celery": {"hpa": {"enabled": True}}, - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-hpa.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels def test_should_remove_replicas_field(self): docs = render_chart( @@ -2607,38 +2662,66 @@ def test_hpa_metrics_override(self, executor, workers_values): class TestWorkerNetworkPolicy: """Tests worker network policy.""" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + { + "labels": {"key": "value"}, + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "networkPolicies": {"enabled": True}, "executor": "CeleryExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-networkpolicy.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels class TestWorkerService: """Tests worker service.""" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + { + "labels": {"key": "value"}, + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-service.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels class TestWorkerCeleryServiceAccount: @@ -2770,20 +2853,40 @@ def test_overwrite_name(self, workers_values): assert jmespath.search("metadata.name", docs[0]) == "test" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": {"serviceAccount": {"create": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + { + "labels": {"key": "value"}, + "celery": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "celery": {"serviceAccount": {"create": True}}, - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels class TestWorkerKubernetesServiceAccount: @@ -2924,17 +3027,37 @@ def test_overwrite_name(self, workers_values): assert jmespath.search("metadata.name", docs[0]) == "test" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "kubernetes": {"serviceAccount": {"create": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "kubernetes": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + { + "labels": {"key": "value"}, + "kubernetes": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": { - "kubernetes": {"serviceAccount": {"create": True}}, - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py index ed2d677fce946..74bb2f6334221 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py @@ -3167,6 +3167,13 @@ def test_overwrite_pod_annotations(self, workers_values): "labels": {"echo": "test"}, "celery": {"enableDefault": False, "sets": [{"name": "set1", "labels": {"test": "echo"}}]}, }, + { + "celery": { + "enableDefault": False, + "labels": {"echo": "test"}, + "sets": [{"name": "set1", "labels": {"test": "echo"}}], + }, + }, ], ) def test_overwrite_labels(self, workers_values):