From 66d53317d56611d83d399e0260cc4dd18ec86e70 Mon Sep 17 00:00:00 2001 From: Pujitha Paladugu <10557236+pujitha24@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:34:54 -0700 Subject: [PATCH 1/2] fix(fluentbit): render namespaced rewrite_tag filter as YAML when configFileFormat=yaml Motivation: When a FluentBitConfig's ClusterFluentBitConfig sets spec.configFileFormat: yaml, and a namespaced Filter/FluentBitConfig triggers the operator's auto-generated `rewrite_tag` filter (used to tag records emitted from a given namespace), the operator always rendered that filter as a hand-written classic TOML snippet (`[Filter]\n Name rewrite_tag\n...`) and spliced it directly into the otherwise-YAML fluent-bit.yaml secret. The result is a config file that mixes TOML and YAML syntax, which fluent-bit fails to parse at startup. This only affects the yaml config format combined with the namespaced rewrite-tag scenario; the classic/TOML format and non-namespaced setups are unaffected, and the operator's reconcile loop itself completes without error since it has no visibility into the malformed content it writes. Fixes #1689 Approach: Instead of hand-building a format-specific string, generateRewriteTagConfig now constructs the existing filter.RewriteTag plugin struct (same Rule/EmitterName/ EmitterStorageType/EmitterMemBufLimit values as before) wrapped in a synthetic ClusterFilterList, and renders it through the same Load()/LoadAsYaml() methods already used for user-defined Filter and ClusterFilter resources. A new configFileFormat parameter is threaded from ClusterFluentBitConfig.Spec.ConfigFileFormat through processNamespacedFluentBitCfgs into generateRewriteTagConfig to pick the right renderer. The classic TOML output is unchanged apart from a harmless reordering of the Emitter_* keys (order doesn't matter in fluent-bit's classic format). Validation: - go build ./... - go test ./controllers/... ./apis/fluentbit/v1alpha2/... (all packages pass, including a new TestGenerateRewriteTagConfigYaml covering both the yaml and classic output paths) - go vet ./... ```release-note Fixed a bug where namespaced Filter resources combined with `configFileFormat: yaml` produced an invalid fluent-bit.yaml config (TOML text mixed into YAML) for the auto-generated rewrite_tag filter, causing fluent-bit to fail to parse its configuration. ``` Signed-off-by: Pujitha Paladugu <10557236+pujitha24@users.noreply.github.com> --- controllers/fluentbitconfig_controller.go | 60 ++++++++----- .../fluentbitconfig_controller_test.go | 86 +++++++++++++++++++ 2 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 controllers/fluentbitconfig_controller_test.go diff --git a/controllers/fluentbitconfig_controller.go b/controllers/fluentbitconfig_controller.go index 2c23e3227..702375393 100644 --- a/controllers/fluentbitconfig_controller.go +++ b/controllers/fluentbitconfig_controller.go @@ -17,7 +17,6 @@ limitations under the License. package controllers import ( - "bytes" "context" "crypto/md5" "crypto/sha256" @@ -26,6 +25,7 @@ import ( "sort" "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/filter" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -241,7 +241,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ nsFilterLists, nsOutputLists, nsParserLists, nsClusterParserLists, nsMultilineParserLists, nsClusterMultilineParserLists, rewriteTagConfigs, err := r.processNamespacedFluentBitCfgs( - ctx, fb, inputs, + ctx, fb, inputs, cfg.Spec.ConfigFileFormat, ) if err != nil { @@ -307,6 +307,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ func (r *FluentBitConfigReconciler) processNamespacedFluentBitCfgs( ctx context.Context, fb fluentbitv1alpha2.FluentBit, inputs fluentbitv1alpha2.ClusterInputList, + configFileFormat *string, ) ( []fluentbitv1alpha2.FilterList, []fluentbitv1alpha2.OutputList, []fluentbitv1alpha2.ParserList, []fluentbitv1alpha2.ClusterParserList, @@ -359,7 +360,12 @@ func (r *FluentBitConfigReconciler) processNamespacedFluentBitCfgs( clusterMultilineParsers = append(clusterMultilineParsers, clusterMultilineParsersList) if _, ok := storeNamespaces[cfg.Namespace]; !ok { - rewriteTagConfig := r.generateRewriteTagConfig(cfg, inputs) + rewriteTagConfig, err := r.generateRewriteTagConfig(cfg, inputs, configFileFormat) + if err != nil { + return filters, outputs, parsers, + clusterParsers, multilineParsers, clusterMultilineParsers, + nil, err + } if rewriteTagConfig != "" { rewriteTagConfigs = append(rewriteTagConfigs, rewriteTagConfig) storeNamespaces[cfg.Namespace] = true @@ -463,8 +469,8 @@ func (r *FluentBitConfigReconciler) ListFluentBitConfigResources( } func (r *FluentBitConfigReconciler) generateRewriteTagConfig( - cfg fluentbitv1alpha2.FluentBitConfig, inputs fluentbitv1alpha2.ClusterInputList, -) string { + cfg fluentbitv1alpha2.FluentBitConfig, inputs fluentbitv1alpha2.ClusterInputList, configFileFormat *string, +) (string, error) { var tag string for _, input := range inputs.Items { if input.Spec.Tail == nil || !strings.Contains(input.Spec.Tail.Path, "/var/log/containers") { @@ -479,28 +485,40 @@ func (r *FluentBitConfigReconciler) generateRewriteTagConfig( } } if tag == "" { - return "" + return "", nil + } + + rewriteTag := &filter.RewriteTag{ + Rules: []string{ + fmt.Sprintf("$kubernetes['namespace_name'] ^(%s)$ %x.$TAG false", cfg.Namespace, md5.Sum([]byte(cfg.Namespace))), + }, } - var buf bytes.Buffer - fmt.Fprintln(&buf, "[Filter]") - fmt.Fprintln(&buf, " Name rewrite_tag") - fmt.Fprintf(&buf, " Match %s\n", tag) - fmt.Fprintf(&buf, " Rule $kubernetes['namespace_name'] ^(%s)$ %x.$TAG false\n", cfg.Namespace, - md5.Sum([]byte(cfg.Namespace))) if cfg.Spec.Service != nil { if cfg.Spec.Service.EmitterName != "" { - fmt.Fprintf(&buf, " Emitter_Name %s\n", cfg.Spec.Service.EmitterName) + rewriteTag.EmitterName = cfg.Spec.Service.EmitterName } else { - fmt.Fprintf(&buf, " Emitter_Name re_emitted_%x\n", md5.Sum([]byte(cfg.Namespace))) - } - if cfg.Spec.Service.EmitterStorageType != "" { - fmt.Fprintf(&buf, " Emitter_Storage.type %s\n", cfg.Spec.Service.EmitterStorageType) - } - if cfg.Spec.Service.EmitterMemBufLimit != "" { - fmt.Fprintf(&buf, " Emitter_Mem_Buf_Limit %s\n", cfg.Spec.Service.EmitterMemBufLimit) + rewriteTag.EmitterName = fmt.Sprintf("re_emitted_%x", md5.Sum([]byte(cfg.Namespace))) } + rewriteTag.EmitterStorageType = cfg.Spec.Service.EmitterStorageType + rewriteTag.EmitterMemBufLimit = cfg.Spec.Service.EmitterMemBufLimit + } + + filterList := fluentbitv1alpha2.ClusterFilterList{ + Items: []fluentbitv1alpha2.ClusterFilter{ + { + Spec: fluentbitv1alpha2.FilterSpec{ + Match: tag, + FilterItems: []fluentbitv1alpha2.FilterItem{{RewriteTag: rewriteTag}}, + }, + }, + }, + } + + sl := plugins.NewSecretLoader(nil, "") + if configFileFormat != nil && *configFileFormat == "yaml" { + return filterList.LoadAsYaml(sl, 1) } - return buf.String() + return filterList.Load(sl) } func (r *FluentBitConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { diff --git a/controllers/fluentbitconfig_controller_test.go b/controllers/fluentbitconfig_controller_test.go new file mode 100644 index 000000000..724d1d74f --- /dev/null +++ b/controllers/fluentbitconfig_controller_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "strings" + "testing" + + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/input" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + fluentbitv1alpha2 "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2" +) + +// TestGenerateRewriteTagConfigYaml reproduces +// https://github.com/fluent/fluent-operator/issues/1689: when +// spec.configFileFormat is "yaml", the auto-generated rewrite_tag filter for +// namespaced FluentBitConfig resources must be rendered as YAML, not as a +// classic TOML snippet spliced into the YAML document. +func TestGenerateRewriteTagConfigYaml(t *testing.T) { + r := &FluentBitConfigReconciler{} + cfg := fluentbitv1alpha2.FluentBitConfig{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foobar"}, + } + inputs := fluentbitv1alpha2.ClusterInputList{ + Items: []fluentbitv1alpha2.ClusterInput{ + { + Spec: fluentbitv1alpha2.InputSpec{ + Tail: &input.Tail{ + Tag: "kube.*", + Path: "/var/log/containers/*.log", + }, + }, + }, + }, + } + + yamlFormat := "yaml" + out, err := r.generateRewriteTagConfig(cfg, inputs, &yamlFormat) + if err != nil { + t.Fatalf("generateRewriteTagConfig returned error: %v", err) + } + + if strings.Contains(out, "[Filter]") { + t.Fatalf("expected YAML output, got classic TOML snippet:\n%s", out) + } + + // The generated snippet must parse as valid YAML on its own, and must be + // usable as a "pipeline.filters" fragment (i.e. it starts with a + // top-level "filters:" key). + var parsed map[string]interface{} + if err := yaml.Unmarshal([]byte(out), &parsed); err != nil { + t.Fatalf("generated rewrite_tag config is not valid YAML: %v\n%s", err, out) + } + if _, ok := parsed["filters"]; !ok { + t.Fatalf("expected a top-level \"filters\" key, got:\n%s", out) + } + + if !strings.Contains(out, "name: rewrite_tag") { + t.Fatalf("expected a rewrite_tag filter entry, got:\n%s", out) + } + + // classic (default) format must remain unchanged (TOML). + classicOut, err := r.generateRewriteTagConfig(cfg, inputs, nil) + if err != nil { + t.Fatalf("generateRewriteTagConfig returned error: %v", err) + } + if !strings.Contains(classicOut, "[Filter]") || !strings.Contains(classicOut, "Name rewrite_tag") { + t.Fatalf("expected classic TOML output, got:\n%s", classicOut) + } +} From 13ebb48d18fc1563a0050e6bad541748fa05f5db Mon Sep 17 00:00:00 2001 From: Pujitha Paladugu <10557236+pujitha24@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:18:35 -0700 Subject: [PATCH 2/2] fix(fluentbit): extract yaml format literal to shared const to satisfy goconst lint Signed-off-by: Pujitha Paladugu <10557236+pujitha24@users.noreply.github.com> --- controllers/consts.go | 2 ++ controllers/fluentbitconfig_controller.go | 4 ++-- controllers/fluentbitconfig_controller_test.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/controllers/consts.go b/controllers/consts.go index c86a9bb0a..e9bd5337f 100644 --- a/controllers/consts.go +++ b/controllers/consts.go @@ -12,4 +12,6 @@ var ( fluentbitApiGVStr = fluentbitv1alpha2.SchemeGroupVersion.String() fluentdApiGVStr = fluentdv1alpha1.SchemeGroupVersion.String() fluentdAgentMode = "agent" + + configFileFormatYaml = "yaml" ) diff --git a/controllers/fluentbitconfig_controller.go b/controllers/fluentbitconfig_controller.go index 702375393..caed63793 100644 --- a/controllers/fluentbitconfig_controller.go +++ b/controllers/fluentbitconfig_controller.go @@ -287,7 +287,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ } configFileName := "fluent-bit.conf" - if cfg.Spec.ConfigFileFormat != nil && *cfg.Spec.ConfigFileFormat == "yaml" { + if cfg.Spec.ConfigFileFormat != nil && *cfg.Spec.ConfigFileFormat == configFileFormatYaml { configFileName = "fluent-bit.yaml" } @@ -515,7 +515,7 @@ func (r *FluentBitConfigReconciler) generateRewriteTagConfig( } sl := plugins.NewSecretLoader(nil, "") - if configFileFormat != nil && *configFileFormat == "yaml" { + if configFileFormat != nil && *configFileFormat == configFileFormatYaml { return filterList.LoadAsYaml(sl, 1) } return filterList.Load(sl) diff --git a/controllers/fluentbitconfig_controller_test.go b/controllers/fluentbitconfig_controller_test.go index 724d1d74f..ac1cdd85b 100644 --- a/controllers/fluentbitconfig_controller_test.go +++ b/controllers/fluentbitconfig_controller_test.go @@ -50,7 +50,7 @@ func TestGenerateRewriteTagConfigYaml(t *testing.T) { }, } - yamlFormat := "yaml" + yamlFormat := configFileFormatYaml out, err := r.generateRewriteTagConfig(cfg, inputs, &yamlFormat) if err != nil { t.Fatalf("generateRewriteTagConfig returned error: %v", err)