system_config: support built-in config files

In the previous versions, if you want to manage multiple
configuration files, you must use @include directive.

In this commit, add an option to specify the directory
(e.g. /etc/fluent/conf.d) which stores additional configuration files.
If there are such files under specified directory, they are loaded by
default without @include directive.

If you want to disable this feature, set empty string
for config_include_dir "".

Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>

Author: kenhys

lib/fluent/config/v1_parser.rb

@@ -16,6 +16,7 @@
 
 require 'strscan'
 require 'uri'
+require 'find'
 
 require 'fluent/config/error'
 require 'fluent/config/basic_parser'
@@ -38,6 +39,7 @@ def initialize(strscan, include_basepath, fname, eval_context)
         @include_basepath = include_basepath
         @fname = fname
         @logger = defined?($log) ? $log : nil
+        @included_config_files = []
       end
 
       def parse!
@@ -96,6 +98,10 @@ def parse_element(root_element, elem_name, attrs = {}, elems = [])
             e_attrs, e_elems = parse_element(false, e_name)
             new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
             new_e.v1_config = true
+            if e_name == "system" and not e_attrs["config_include_dir"]&.empty?
+              # include additional configuration files
+              include_additional_config_files(attrs, elems, e_attrs["config_include_dir"])
+            end
             elems << new_e
 
           elsif root_element && skip(/(\@include|include)#{SPACING}/)
@@ -143,6 +149,9 @@ def parse_element(root_element, elem_name, attrs = {}, elems = [])
 
       def parse_include(attrs, elems)
         uri = scan_string(LINE_END)
+        if @included_config_files.include?(File.expand_path(File.join(@include_basepath, uri)))
+          return
+        end
         eval_include(attrs, elems, uri)
         line_end
       end
@@ -183,6 +192,30 @@ def eval_include(attrs, elems, uri)
         raise cpe
       end
 
+      def include_additional_config_files(attrs, elems, config_dir)
+        if config_dir.empty?
+          @logger.info "disable additional config directory" if @logger
+          return
+        end
+        unless File.directory?(config_dir)
+          @logger.warn "no valid config directory: <#{config_dir}>" if @logger
+          return
+        end
+        Find.find("#{config_dir}") do |path|
+          next if File.directory?(path)
+          unless path.end_with?(".conf")
+            @logger.debug "ignore non config file: #{path}" if @logger
+            next
+          end
+          expanded_config_path = File.expand_path(path)
+          unless @included_config_files.include?(expanded_config_path)
+            @included_config_files << expanded_config_path
+            @logger.info "include additional #{expanded_config_path}" if @logger
+            eval_include(attrs, elems, expanded_config_path)
+          end
+        end
+      end
+
       # override
       def error_sample
         "#{@fname} #{super}"

lib/fluent/env.rb

@@ -24,6 +24,7 @@ module Fluent
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
+  DEFAULT_CONFIG_DIR = ENV['FLUENT_CONF_DIR'] || '/etc/fluent/conf.d'
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644

lib/fluent/system_config.rb

@@ -28,7 +28,8 @@ class SystemConfig
       :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,
+      :config_include_dir
     ]
 
     config_param :workers,   :integer, default: 1
@@ -58,6 +59,7 @@ class SystemConfig
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_param :config_include_dir, default: ENV["FLUENT_CONF_DIR"] || '/etc/fluent/conf.d'
     config_section :log, required: false, init: true, multi: false do
       config_param :path, :string, default: nil
       config_param :format, :enum, list: [:text, :json], default: :text