system_config: support built-in config files

In the previous versions, if you want to manage multiple
configuration files, you must use @include directive.

In this commit, add an option to specify the directory
(e.g. /etc/fluent/conf.d) which stores additional configuration files.
If there are such files under specified directory, they are loaded by
default without @include directive.

If you want to disable this feature, set empty string
for config_include_dir "".

Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>

Author: kenhys

lib/fluent/config/element.rb

@@ -71,6 +71,12 @@ def add_element(name, arg = '')
         e
       end
 
+      def merge_elements(elements)
+        elements.each do |e|
+          @elements << e
+        end
+      end
+
       def inspect
         attrs = super
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect

lib/fluent/env.rb

@@ -24,6 +24,7 @@ module Fluent
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
+  DEFAULT_CONFIG_DIR = ENV['FLUENT_CONF_DIR'] || '/etc/fluent/conf.d'
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644

lib/fluent/supervisor.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'open3'
 require 'pathname'
+require 'find'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -806,6 +807,26 @@ def configure(supervisor: false)
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
 
+      # merge additional configuration
+      config_type = if File.extname(@config_path) == '.yaml' || File.extname(@config_path) == '.yml'
+                      :yaml
+                    else
+                      nil
+                    end
+      Find.find(@system_config.config_include_dir) do |path|
+        next if File.directory?(path)
+        if config_type == :yaml
+          next if File.extname(path) != '.yaml' || File.extname(path) != '.yml'
+        else
+          next if File.extname(path) == '.yaml' || File.extname(path) == '.yml'
+        end
+        elements = Fluent::Config.build(config_path: path,
+                                        encoding: @conf_encoding,
+                                        use_v1_config: @use_v1_config,
+                                        type: config_type).elements
+        @conf.merge_elements(elements)
+      end
+
       @libs.each do |lib|
         require lib
       end

lib/fluent/system_config.rb

@@ -28,7 +28,8 @@ class SystemConfig
       :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,
+      :config_include_dir
     ]
 
     config_param :workers,   :integer, default: 1
@@ -58,6 +59,7 @@ class SystemConfig
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_param :config_include_dir, default: ENV["FLUENT_CONF_DIR"] || '/etc/fluent/conf.d'
     config_section :log, required: false, init: true, multi: false do
       config_param :path, :string, default: nil
       config_param :format, :enum, list: [:text, :json], default: :text