From aaa46cd74be291a90330b378d485bc15271bbdbf Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Fri, 4 Apr 2025 18:18:16 +0900 Subject: [PATCH 1/5] system_config: support built-in config files Currently @include directive supports to reuse configuration files, but no way to apply specific configuration files by default (without specifying to). In the previous versions, if you want to manage multiple configuration files, you must use @include directive explicitly. In this commit, add an option to specify the directory (e.g. /etc/fluent/conf.d) which stores additional configuration files. If there are such files under specified directory, they are loaded by default without @include directive. If you want to disable this feature, set empty string for config_include_dir "". Signed-off-by: Kentaro Hayashi --- lib/fluent/env.rb | 1 + lib/fluent/supervisor.rb | 31 +++++++++ lib/fluent/system_config.rb | 5 +- test/command/test_fluentd.rb | 72 ++++++++++++++++++++ test/test_supervisor.rb | 123 +++++++++++++++++++++++++++++++++++ 5 files changed, 231 insertions(+), 1 deletion(-) diff --git a/lib/fluent/env.rb b/lib/fluent/env.rb index 1b6c38384f..33aaa7223c 100644 --- a/lib/fluent/env.rb +++ b/lib/fluent/env.rb @@ -21,6 +21,7 @@ module Fluent DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf' + DEFAULT_CONFIG_INCLUDE_DIR = ENV["FLUENT_CONF_INCLUDE_DIR"] || '/etc/fluent/conf.d' DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin' DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock' DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent' diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 246c5ed586..0132ba3e3b 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -17,6 +17,7 @@ require 'fileutils' require 'open3' require 'pathname' +require 'find' require 'fluent/config' require 'fluent/counter' @@ -806,6 +807,10 @@ def configure(supervisor: false) $log.info :supervisor, 'parsing config file is succeeded', path: @config_path + build_additional_configurations do |additional_conf| + @conf += additional_conf + end + @libs.each do |lib| require lib end @@ -1090,6 +1095,10 @@ def reload_config type: @config_file_type, ) + build_additional_configurations do |additional_conf| + conf += additional_conf + end + Fluent::VariableStore.try_to_reset do Fluent::Engine.reload_config(conf) end @@ -1196,6 +1205,28 @@ def build_system_config(conf) system_config end + def build_additional_configurations + if @system_config.config_include_dir&.empty? + $log.info :supervisor, 'default configuration include directory was disabled' + return + end + begin + Find.find(@system_config.config_include_dir) do |path| + next if File.directory?(path) + next unless [".conf", ".yaml", ".yml"].include?(File.extname(path)) + # NOTE: both types of normal config (.conf) and YAML will be loaded. + # Thus, it does not care whether @config_path is .conf or .yml. + $log.info :supervisor, 'loading additional configuration file', path: path + yield Fluent::Config.build(config_path: path, + encoding: @conf_encoding, + use_v1_config: @use_v1_config, + type: :guess) + end + rescue Errno::ENOENT + $log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir + end + end + RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze def build_spawn_command diff --git a/lib/fluent/system_config.rb b/lib/fluent/system_config.rb index 6e280011aa..bdf94e4728 100644 --- a/lib/fluent/system_config.rb +++ b/lib/fluent/system_config.rb @@ -16,6 +16,7 @@ require 'fluent/configurable' require 'fluent/config/element' +require 'fluent/env' module Fluent class SystemConfig @@ -28,7 +29,8 @@ class SystemConfig :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission, :counter_server, :counter_client, :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket, - :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer + :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer, + :config_include_dir ] config_param :workers, :integer, default: 1 @@ -58,6 +60,7 @@ class SystemConfig config_param :dir_permission, default: nil do |v| v.to_i(8) end + config_param :config_include_dir, default: Fluent::DEFAULT_CONFIG_INCLUDE_DIR config_section :log, required: false, init: true, multi: false do config_param :path, :string, default: nil config_param :format, :enum, list: [:text, :json], default: :text diff --git a/test/command/test_fluentd.rb b/test/command/test_fluentd.rb index 15cbd08c95..98d67108cf 100644 --- a/test/command/test_fluentd.rb +++ b/test/command/test_fluentd.rb @@ -1536,4 +1536,76 @@ def send_end(port) end_thread&.kill end end + + def create_config_include_dir_configuration(config_path, config_dir, yaml_format = false) + if yaml_format + conf = < + config_include_dir #{config_dir} + +CONF + end + create_conf_file(config_path, conf) + end + + sub_test_case "test additional configuration directory" do + setup do + FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d")) + end + + test "disable additional configuration directory" do + conf_path = create_config_include_dir_configuration("disabled_config_include_dir.conf", "") + assert_log_matches(create_cmdline(conf_path), + "[info]: default configuration include directory was disabled") + end + + test "inaccessible include directory error" do + conf_path = create_config_include_dir_configuration("inaccessible_include.conf", "/nonexistent") + assert_log_matches(create_cmdline(conf_path), + "[info]: inaccessible include directory was specified") + end + + data("include additional configuration with relative conf.d" => [{"relative_path" => true}], + "include additional configuration with full-path conf.d" => [{"relative_path" => false}]) + test "additional configuration file (conf.d/child.conf) was loaded" do |(option)| + conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d" + conf_path = create_config_include_dir_configuration("parent.conf", conf_dir) + create_conf_file('conf.d/child.conf', "") + assert_log_matches(create_cmdline(conf_path), + "[info]: loading additional configuration file path=\"#{conf_dir}/child.conf\"") + end + end + + sub_test_case "test additional configuration directory (YAML)" do + setup do + FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d")) + end + + test "disable additional configuration directory" do + conf_path = create_config_include_dir_configuration("disabled_config_include_dir.yml", "", true) + assert_log_matches(create_cmdline(conf_path), + "[info]: default configuration include directory was disabled") + end + + test "inaccessible include directory error" do + conf_path = create_config_include_dir_configuration("inaccessible_include.yml", "/nonexistent", true) + assert_log_matches(create_cmdline(conf_path), + "[info]: inaccessible include directory was specified") + end + + data("include additional YAML configuration with relative conf.d" => [{"relative_path" => true}], + "include additional YAML configuration with full path conf.d" => [{"relative_path" => false}]) + test "additional relative configuration file (conf.d/child.yml) was loaded" do |(option)| + conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d" + conf_path = create_config_include_dir_configuration("parent.yml", conf_dir, true) + create_conf_file('conf.d/child.yml', "") + assert_log_matches(create_cmdline(conf_path), + "[info]: loading additional configuration file path=\"#{conf_dir}/child.yml\"") + end + end end diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 82cbf71321..173d891733 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -1054,6 +1054,129 @@ def test_stop_parallel_old_supervisor_after_delay end end + sub_test_case "include additional configuration" do + setup do + @config_include_dir = File.join(@tmp_dir, "conf.d") + FileUtils.mkdir_p(@config_include_dir) + end + + test "no additional configuration" do + c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => '' }, []) + stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) } + supervisor = Fluent::Supervisor.new({}) + stub(supervisor).build_spawn_command { "dummy command line" } + supervisor.configure(supervisor: true) + assert_equal([c], supervisor.instance_variable_get(:@conf).elements) + end + + data( + "single source" => ["forward"], + "multiple sources" => ["forward", "tcp"]) + test "additional configuration" do |sources| + c = Fluent::Config::Element.new('system', '', + { 'config_include_dir' => @config_include_dir }, []) + config_path = "#{@config_include_dir}/dummy.conf" + stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8", + additional_config: anything, use_v1_config: anything, + type: anything) { config_element('ROOT', '', {}, [c]) } + sources.each do |type| + config = <<~EOF + + @type #{type} + + EOF + additional_config_path = "#{@config_include_dir}/#{type}.yml" + write_config(additional_config_path, config) + stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", + use_v1_config: true, type: :guess) { + Fluent::Config.parse(File.read(additional_config_path), File.dirname(config_path), true) + } + end + supervisor = Fluent::Supervisor.new({}) + stub(supervisor).build_spawn_command { "dummy command line" } + supervisor.configure(supervisor: true) + expected = [c].concat(sources.collect { |type| {"@type" => type} }) + assert_equal(expected, supervisor.instance_variable_get(:@conf).elements) + end + + data( + "single YAML source" => ["forward"], + "multiple YAML sources" => ["forward", "tcp"]) + test "additional YAML configuration" do |sources| + c = Fluent::Config::Element.new('system', '', + { 'config_include_dir' => @config_include_dir }, []) + config_path = "#{@config_include_dir}/dummy.yml" + stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8", + additional_config: anything, use_v1_config: anything, + type: anything) { config_element('ROOT', '', {}, [c]) } + sources.each do |type| + config = <<~EOF + config: + - source: + $type: #{type} + EOF + additional_config_path = "#{@config_include_dir}/#{type}.yml" + write_config(additional_config_path, config) + stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", + use_v1_config: true, type: :guess) { + Fluent::Config::YamlParser.parse(additional_config_path) + } + end + supervisor = Fluent::Supervisor.new({}) + stub(supervisor).build_spawn_command { "dummy command line" } + supervisor.configure(supervisor: true) + expected = [c].concat(sources.collect { |type| {"@type" => type} }) + assert_equal(expected, supervisor.instance_variable_get(:@conf).elements) + end + + data( + "single source" => [false, ["forward"]], + "multiple sources" => [false, ["forward", "tcp"]], + "single YAML source" => [true, ["forward"]], + "multiple YAML sources" => [true, ["forward", "tcp"]]) + test "reload with additional configuration" do |(yaml, sources)| + c = Fluent::Config::Element.new('system', '', + { 'config_include_dir' => @config_include_dir }, []) + config_path = "#{@config_include_dir}/dummy.yml" + stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8", + additional_config: anything, use_v1_config: anything, + type: anything) { config_element('ROOT', '', {}, [c]) } + sources.each do |type| + if yaml + config = <<~EOF + config: + - source: + $type: #{type} + EOF + additional_config_path = "#{@config_include_dir}/#{type}.yml" + write_config(additional_config_path, config) + stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", + use_v1_config: true, type: :guess) { + Fluent::Config::YamlParser.parse(additional_config_path) + } + else + config = <<~EOF + + @type #{type} + + EOF + additional_config_path = "#{@config_include_dir}/#{type}.conf" + write_config(additional_config_path, config) + stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", + use_v1_config: true, type: :guess) { + Fluent::Config.parse(File.read(additional_config_path), File.dirname(config_path), true) + } + end + end + supervisor = Fluent::Supervisor.new({}) + stub(supervisor).build_spawn_command { "dummy command line" } + supervisor.configure(supervisor: true) + supervisor.__send__(:reload_config) + expected = [c].concat(sources.collect { |type| {"@type" => type} }) + assert_equal(expected, supervisor.instance_variable_get(:@conf).elements) + end + end + def create_debug_dummy_logger dl_opts = {} dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG From e3922c1aefccc5a8ae03a11842f3da7af0992ae7 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Tue, 22 Apr 2025 17:22:20 +0900 Subject: [PATCH 2/5] Update lib/fluent/supervisor.rb Co-authored-by: Daijiro Fukuda Signed-off-by: Kentaro Hayashi --- lib/fluent/supervisor.rb | 8 ++++---- test/command/test_fluentd.rb | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 0132ba3e3b..adc517610c 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -1207,7 +1207,7 @@ def build_system_config(conf) def build_additional_configurations if @system_config.config_include_dir&.empty? - $log.info :supervisor, 'default configuration include directory was disabled' + $log.info :supervisor, 'configuration include directory is disabled' return end begin @@ -1218,9 +1218,9 @@ def build_additional_configurations # Thus, it does not care whether @config_path is .conf or .yml. $log.info :supervisor, 'loading additional configuration file', path: path yield Fluent::Config.build(config_path: path, - encoding: @conf_encoding, - use_v1_config: @use_v1_config, - type: :guess) + encoding: @conf_encoding, + use_v1_config: @use_v1_config, + type: :guess) end rescue Errno::ENOENT $log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir diff --git a/test/command/test_fluentd.rb b/test/command/test_fluentd.rb index 98d67108cf..761c983af8 100644 --- a/test/command/test_fluentd.rb +++ b/test/command/test_fluentd.rb @@ -1570,9 +1570,9 @@ def create_config_include_dir_configuration(config_path, config_dir, yaml_format "[info]: inaccessible include directory was specified") end - data("include additional configuration with relative conf.d" => [{"relative_path" => true}], - "include additional configuration with full-path conf.d" => [{"relative_path" => false}]) - test "additional configuration file (conf.d/child.conf) was loaded" do |(option)| + data("include additional configuration with relative conf.d" => {"relative_path" => true}, + "include additional configuration with full-path conf.d" => {"relative_path" => false}) + test "additional configuration file (conf.d/child.conf) was loaded" do |option| conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d" conf_path = create_config_include_dir_configuration("parent.conf", conf_dir) create_conf_file('conf.d/child.conf', "") @@ -1598,9 +1598,9 @@ def create_config_include_dir_configuration(config_path, config_dir, yaml_format "[info]: inaccessible include directory was specified") end - data("include additional YAML configuration with relative conf.d" => [{"relative_path" => true}], - "include additional YAML configuration with full path conf.d" => [{"relative_path" => false}]) - test "additional relative configuration file (conf.d/child.yml) was loaded" do |(option)| + data("include additional YAML configuration with relative conf.d" => {"relative_path" => true}, + "include additional YAML configuration with full path conf.d" => {"relative_path" => false}) + test "additional relative configuration file (conf.d/child.yml) was loaded" do |option| conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d" conf_path = create_config_include_dir_configuration("parent.yml", conf_dir, true) create_conf_file('conf.d/child.yml', "") From d50df85126364554fac6177f33b003fcc07f9d46 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Tue, 22 Apr 2025 18:26:55 +0900 Subject: [PATCH 3/5] test: omit needless stub build_spawn_command Signed-off-by: Kentaro Hayashi --- test/test_supervisor.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 173d891733..2dc16190c7 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -1064,7 +1064,6 @@ def test_stop_parallel_old_supervisor_after_delay c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => '' }, []) stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) } supervisor = Fluent::Supervisor.new({}) - stub(supervisor).build_spawn_command { "dummy command line" } supervisor.configure(supervisor: true) assert_equal([c], supervisor.instance_variable_get(:@conf).elements) end @@ -1093,7 +1092,6 @@ def test_stop_parallel_old_supervisor_after_delay } end supervisor = Fluent::Supervisor.new({}) - stub(supervisor).build_spawn_command { "dummy command line" } supervisor.configure(supervisor: true) expected = [c].concat(sources.collect { |type| {"@type" => type} }) assert_equal(expected, supervisor.instance_variable_get(:@conf).elements) @@ -1123,7 +1121,6 @@ def test_stop_parallel_old_supervisor_after_delay } end supervisor = Fluent::Supervisor.new({}) - stub(supervisor).build_spawn_command { "dummy command line" } supervisor.configure(supervisor: true) expected = [c].concat(sources.collect { |type| {"@type" => type} }) assert_equal(expected, supervisor.instance_variable_get(:@conf).elements) @@ -1169,7 +1166,6 @@ def test_stop_parallel_old_supervisor_after_delay end end supervisor = Fluent::Supervisor.new({}) - stub(supervisor).build_spawn_command { "dummy command line" } supervisor.configure(supervisor: true) supervisor.__send__(:reload_config) expected = [c].concat(sources.collect { |type| {"@type" => type} }) From a336794106e1479318eb76762b3508a31819cddb Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Tue, 22 Apr 2025 18:39:04 +0900 Subject: [PATCH 4/5] test: follow changed info message Signed-off-by: Kentaro Hayashi --- test/command/test_fluentd.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/command/test_fluentd.rb b/test/command/test_fluentd.rb index 761c983af8..ec280e7253 100644 --- a/test/command/test_fluentd.rb +++ b/test/command/test_fluentd.rb @@ -1561,7 +1561,7 @@ def create_config_include_dir_configuration(config_path, config_dir, yaml_format test "disable additional configuration directory" do conf_path = create_config_include_dir_configuration("disabled_config_include_dir.conf", "") assert_log_matches(create_cmdline(conf_path), - "[info]: default configuration include directory was disabled") + "[info]: configuration include directory is disabled") end test "inaccessible include directory error" do @@ -1589,7 +1589,7 @@ def create_config_include_dir_configuration(config_path, config_dir, yaml_format test "disable additional configuration directory" do conf_path = create_config_include_dir_configuration("disabled_config_include_dir.yml", "", true) assert_log_matches(create_cmdline(conf_path), - "[info]: default configuration include directory was disabled") + "[info]: configuration include directory is disabled") end test "inaccessible include directory error" do From 9b9038d39b1de8512f22da60af56284be625517b Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Wed, 23 Apr 2025 12:07:02 +0900 Subject: [PATCH 5/5] test: drop needless stub Before: stub all Fluent::Config.build After stub first Fluent::Config.build, do not stub additional Fluent::Config.build Signed-off-by: Kentaro Hayashi --- test/test_supervisor.rb | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 2dc16190c7..fd89ca8ce4 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -1075,6 +1075,7 @@ def test_stop_parallel_old_supervisor_after_delay c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => @config_include_dir }, []) config_path = "#{@config_include_dir}/dummy.conf" + stub.proxy(Fluent::Config).build stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8", additional_config: anything, use_v1_config: anything, type: anything) { config_element('ROOT', '', {}, [c]) } @@ -1084,12 +1085,8 @@ def test_stop_parallel_old_supervisor_after_delay @type #{type} EOF - additional_config_path = "#{@config_include_dir}/#{type}.yml" + additional_config_path = "#{@config_include_dir}/#{type}.conf" write_config(additional_config_path, config) - stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", - use_v1_config: true, type: :guess) { - Fluent::Config.parse(File.read(additional_config_path), File.dirname(config_path), true) - } end supervisor = Fluent::Supervisor.new({}) supervisor.configure(supervisor: true) @@ -1104,6 +1101,7 @@ def test_stop_parallel_old_supervisor_after_delay c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => @config_include_dir }, []) config_path = "#{@config_include_dir}/dummy.yml" + stub.proxy(Fluent::Config).build stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8", additional_config: anything, use_v1_config: anything, type: anything) { config_element('ROOT', '', {}, [c]) } @@ -1115,10 +1113,6 @@ def test_stop_parallel_old_supervisor_after_delay EOF additional_config_path = "#{@config_include_dir}/#{type}.yml" write_config(additional_config_path, config) - stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", - use_v1_config: true, type: :guess) { - Fluent::Config::YamlParser.parse(additional_config_path) - } end supervisor = Fluent::Supervisor.new({}) supervisor.configure(supervisor: true) @@ -1135,6 +1129,7 @@ def test_stop_parallel_old_supervisor_after_delay c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => @config_include_dir }, []) config_path = "#{@config_include_dir}/dummy.yml" + stub.proxy(Fluent::Config).build stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8", additional_config: anything, use_v1_config: anything, type: anything) { config_element('ROOT', '', {}, [c]) } @@ -1147,10 +1142,6 @@ def test_stop_parallel_old_supervisor_after_delay EOF additional_config_path = "#{@config_include_dir}/#{type}.yml" write_config(additional_config_path, config) - stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", - use_v1_config: true, type: :guess) { - Fluent::Config::YamlParser.parse(additional_config_path) - } else config = <<~EOF @@ -1159,10 +1150,6 @@ def test_stop_parallel_old_supervisor_after_delay EOF additional_config_path = "#{@config_include_dir}/#{type}.conf" write_config(additional_config_path, config) - stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8", - use_v1_config: true, type: :guess) { - Fluent::Config.parse(File.read(additional_config_path), File.dirname(config_path), true) - } end end supervisor = Fluent::Supervisor.new({})