From 25dac2c32107e1d6364ae6869d2c35a7a4ca3a08 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Sun, 19 Jul 2026 22:25:18 +0530 Subject: [PATCH] parser_syslog: accept a space after RFC3164 priority Signed-off-by: Akash Kumar --- lib/fluent/plugin/parser_syslog.rb | 81 ++++++++++- test/plugin/test_in_syslog.rb | 72 ++++++++++ test/plugin/test_parser_syslog.rb | 214 +++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 6 deletions(-) diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb index 2b6fcca479..4bbc41b89c 100644 --- a/lib/fluent/plugin/parser_syslog.rb +++ b/lib/fluent/plugin/parser_syslog.rb @@ -71,6 +71,7 @@ def initialize @time_parser_rfc5424 = nil @space_count_rfc3164 = nil @space_count_rfc5424 = nil + @time_format_starts_with_space_rfc3164 = false @skip_space_count_rfc3164 = false @skip_space_count_rfc5424 = false @time_parser_rfc5424_without_subseconds = nil @@ -123,10 +124,12 @@ class << self def setup_time_parser_3164(time_fmt) @time_parser_rfc3164 = time_parser_create(format: time_fmt) + @time_format_starts_with_space_rfc3164 = time_fmt.start_with?(SPLIT_CHAR) if ['%b %d %H:%M:%S', '%b %d %H:%M:%S.%N'].include?(time_fmt) @skip_space_count_rfc3164 = true end @space_count_rfc3164 = time_fmt.squeeze(' ').count(' ') + 1 + @space_count_rfc3164 -= 1 if @time_format_starts_with_space_rfc3164 end def setup_time_parser_5424(time_fmt) @@ -162,6 +165,13 @@ def parse_auto(text, &block) end SPLIT_CHAR = ' '.freeze + SPLIT_BYTE = 0x20 + DOT_CHAR = '.'.freeze + DOT_BYTE = 0x2e + GREATER_THAN_BYTE = 0x3e + ZERO_BYTE = 0x30 + NINE_BYTE = 0x39 + RFC3164_PRI_MINIMUM = [nil, nil, nil, 1, 10, 100].freeze def parse_rfc3164_regex(text, &block) idx = 0 @@ -180,12 +190,21 @@ def parse_rfc3164_regex(text, &block) i = idx - 1 sq = false + first_time_field = true @space_count_rfc3164.times do while text[i + 1] == SPLIT_CHAR + if first_time_field + unless @time_format_starts_with_space_rfc3164 + idx += 1 + i += 1 + break + end + end sq = true i += 1 end + first_time_field = false i = text.index(SPLIT_CHAR, i + 1) end @@ -287,14 +306,43 @@ def parse_rfc3164(text, &block) end end + byte = text.getbyte(cursor) + space_after_priority = if byte == SPLIT_BYTE + true + elsif @with_priority && text.getbyte(cursor - 1) != GREATER_THAN_BYTE + # String#index returns a character index. Mark + # non-ASCII PRI for the same timestamp lookup. + byte = nil + text[cursor] == SPLIT_CHAR + else + false + end + if space_after_priority + cursor = rfc3164_space_cursor(text, cursor, pri) + unless cursor + yield nil, nil + return + end + end + if @skip_space_count_rfc3164 # header part time_size = 15 # skip Mmm dd hh:mm:ss - time_end = text[cursor + time_size] - if time_end == SPLIT_CHAR + time_end_index = cursor + time_size + if byte + time_end = text.getbyte(time_end_index) + split_delimiter = SPLIT_BYTE + dot_delimiter = DOT_BYTE + else + time_end = text[time_end_index] + split_delimiter = SPLIT_CHAR + dot_delimiter = DOT_CHAR + end + + if time_end == split_delimiter time_str = text.slice(cursor, time_size) cursor += 16 # time + ' ' - elsif time_end == '.'.freeze + elsif time_end == dot_delimiter # support subsecond time i = text.index(SPLIT_CHAR, time_size) time_str = text.slice(cursor, i - cursor) @@ -314,12 +362,12 @@ def parse_rfc3164(text, &block) i = text.index(SPLIT_CHAR, i + 1) end - time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor) + time_str = sq ? text.slice(cursor, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor) cursor = i + 1 end i = text.index(SPLIT_CHAR, cursor) - if i.nil? + unless i yield nil, nil return end @@ -333,7 +381,7 @@ def parse_rfc3164(text, &block) i = text.index(SPLIT_CHAR, cursor) # message part - msg = if i.nil? # for 'only non-space content case' + msg = unless i # for 'only non-space content case' text.slice(cursor, text.bytesize) else if text[i - 1] == ':'.freeze @@ -369,6 +417,27 @@ def parse_rfc3164(text, &block) yield time, record end + def rfc3164_space_cursor(text, cursor, pri) + if @with_priority + # A nonzero value with the expected decimal width proves that the + # whole PRI is numeric. Leading-zero values take the bytewise path. + minimum = RFC3164_PRI_MINIMUM[cursor] + return unless minimum + + unless pri >= minimum + priority_byte = 1 + while priority_byte < cursor - 1 + byte = text.getbyte(priority_byte) + return unless byte >= ZERO_BYTE && byte <= NINE_BYTE + + priority_byte += 1 + end + end + end + + @time_format_starts_with_space_rfc3164 ? cursor : cursor + 1 + end + NILVALUE = '-'.freeze def parse_rfc5424(text, &block) diff --git a/test/plugin/test_in_syslog.rb b/test/plugin/test_in_syslog.rb index 868e77bb04..09b5136524 100755 --- a/test/plugin/test_in_syslog.rb +++ b/test/plugin/test_in_syslog.rb @@ -124,6 +124,78 @@ def test_time_format(data) } end + data( + 'regexp/rfc3164/parser priority' => ['regexp', 'rfc3164', true], + 'string/rfc3164/parser priority' => ['string', 'rfc3164', true], + 'regexp/auto/parser priority' => ['regexp', 'auto', true], + 'string/auto/parser priority' => ['string', 'auto', true], + 'regexp/rfc3164/input priority' => ['regexp', 'rfc3164', false], + 'string/rfc3164/input priority' => ['string', 'rfc3164', false], + 'regexp/auto/input priority' => ['regexp', 'auto', false], + 'string/auto/input priority' => ['string', 'auto', false], + ) + def test_space_between_rfc3164_priority_and_header(data) + parser_engine, message_format, with_priority = data + d = create_driver([ + ipv4_config, + 'severity_key severity', + 'facility_key facility', + '', + " parser_engine #{parser_engine}", + " message_format #{message_format}", + " with_priority #{with_priority}", + '', + ].join("\n")) + + message = 'Apr 25 16:43:29 PAA-SW1-1 General[procLOG]: main.c(257) 272264 %% Stopping System API application' + d.run(expect_emits: 2) do + u = UDPSocket.new + u.connect('127.0.0.1', @port) + u.send("<14>#{message}", 0) + u.send("<14> #{message}", 0) + end + + assert_equal(2, d.events.size) + assert_equal(d.events[0][1], d.events[1][1]) + assert_equal(d.events[0][2], d.events[1][2]) + d.events.each do |tag, time, record| + assert_equal('syslog.user.info', tag) + assert_equal(event_time('Apr 25 16:43:29', format: '%b %d %H:%M:%S'), time) + assert_equal('user', record['facility']) + assert_equal('info', record['severity']) + assert_equal('PAA-SW1-1', record['host']) + assert_equal('General', record['ident']) + assert_equal('main.c(257) 272264 %% Stopping System API application', record['message']) + end + end + + data('regexp' => 'regexp', 'string' => 'string') + def test_space_after_input_owned_priority_preserves_input_priority_syntax(parser_engine) + d = create_driver([ + ipv4_config, + 'emit_unmatched_lines true', + '', + " parser_engine #{parser_engine}", + ' with_priority false', + '', + ].join("\n")) + + messages = [ + '<1234>Apr 25 16:43:29 host app: message', + '<1234> Apr 25 16:43:29 host app: message', + ' Apr 25 16:43:29 host app: message', + ] + d.run(expect_emits: 3) do + u = UDPSocket.new + u.connect('127.0.0.1', @port) + messages.each { |message| u.send(message, 0) } + end + + assert_equal(d.events[0], d.events[1]) + assert_equal('syslog.unmatched', d.events[2][0]) + assert_equal(messages[2], d.events[2][2]['unmatched_line']) + end + def test_msg_size d = create_driver tests = create_test_case diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb index dcdbca8388..1eb884ce7f 100644 --- a/test/plugin/test_parser_syslog.rb +++ b/test/plugin/test_parser_syslog.rb @@ -75,6 +75,220 @@ def test_parse_with_priority(param) assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format']) end + data( + 'regexp/rfc3164/parser priority' => ['regexp', 'rfc3164', true], + 'string/rfc3164/parser priority' => ['string', 'rfc3164', true], + 'regexp/auto/parser priority' => ['regexp', 'auto', true], + 'string/auto/parser priority' => ['string', 'auto', true], + 'regexp/rfc3164/without priority' => ['regexp', 'rfc3164', false], + 'string/rfc3164/without priority' => ['string', 'rfc3164', false], + 'regexp/auto/without priority' => ['regexp', 'auto', false], + 'string/auto/without priority' => ['string', 'auto', false], + ) + def test_parse_with_space_between_rfc3164_priority_and_header(data) + parser_engine, message_format, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'message_format' => message_format, + 'with_priority' => with_priority, + 'keep_time_key' => true, + ) + + prefix = with_priority ? '<14>' : '' + message = 'Apr 25 16:43:29 PAA-SW1-1 General[procLOG]: main.c(257) 272264 %% Stopping System API application' + results = ["#{prefix}#{message}", "#{prefix} #{message}"].map do |text| + result = nil + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + result + end + + assert_equal(results[0], results[1]) + time, record = results[1] + assert_equal(event_time('Apr 25 16:43:29', format: '%b %d %H:%M:%S'), time) + assert_equal(14, record['pri']) if with_priority + assert_equal('Apr 25 16:43:29', record['time']) + assert_equal('PAA-SW1-1', record['host']) + assert_equal('General', record['ident']) + assert_equal('main.c(257) 272264 %% Stopping System API application', record['message']) + end + + data( + 'regexp/single digit day/with priority' => ['regexp', true, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'string/single digit day/with priority' => ['string', true, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'regexp/subseconds/with priority' => ['regexp', true, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'string/subseconds/with priority' => ['string', true, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'regexp/custom format/with priority' => ['regexp', true, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + 'string/custom format/with priority' => ['string', true, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + 'regexp/single digit day/without priority' => ['regexp', false, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'string/single digit day/without priority' => ['string', false, '%b %d %H:%M:%S', 'Apr 5 16:43:29'], + 'regexp/subseconds/without priority' => ['regexp', false, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'string/subseconds/without priority' => ['string', false, '%b %d %H:%M:%S.%N', 'Apr 25 16:43:29.123456789'], + 'regexp/custom format/without priority' => ['regexp', false, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + 'string/custom format/without priority' => ['string', false, '%Y-%m-%dT%H:%M:%S', '2026-04-25T16:43:29'], + ) + def test_parse_rfc3164_time_variants_with_space_after_priority(data) + parser_engine, with_priority, time_format, timestamp = data + @parser.configure( + 'parser_engine' => parser_engine, + 'time_format' => time_format, + 'with_priority' => with_priority, + 'keep_time_key' => true, + ) + + suffix = "#{timestamp} host app[123]: message" + prefix = with_priority ? '<14>' : '' + results = ["#{prefix}#{suffix}", "#{prefix} #{suffix}"].map do |text| + result = nil + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + result + end + + assert_equal(results[0], results[1]) + expected_time_key = parser_engine == 'regexp' ? timestamp.squeeze(' ') : timestamp + assert_equal(expected_time_key, results[1][1]['time']) + assert_equal('host', results[1][1]['host']) + assert_equal('app', results[1][1]['ident']) + assert_equal('123', results[1][1]['pid']) + assert_equal('message', results[1][1]['message']) + end + + data( + 'regexp/parser priority' => ['regexp', true], + 'string/parser priority' => ['string', true], + 'regexp/input priority' => ['regexp', false], + 'string/input priority' => ['string', false], + ) + def test_parse_with_leading_space_time_format(data) + parser_engine, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'time_format' => ' %b %d %H:%M:%S', + 'with_priority' => with_priority, + 'keep_time_key' => true, + ) + + prefix = with_priority ? '<14>' : '' + result = nil + @parser.instance.parse("#{prefix} Apr 25 16:43:29 host app[123]: message") do |time, record| + result = [time, record] + end + + time, record = result + assert_equal(event_time('Apr 25 16:43:29', format: '%b %d %H:%M:%S'), time) + assert_equal(14, record['pri']) if with_priority + assert_equal(' Apr 25 16:43:29', record['time']) + assert_equal( + {'host' => 'host', 'ident' => 'app', 'pid' => '123', 'message' => 'message'}, + record.reject { |key, _| ['pri', 'time'].include?(key) }, + ) + end + + data( + 'regexp/two spaces' => ['regexp', true, '<14> Apr 25 16:43:29 host app: message'], + 'string/two spaces' => ['string', true, '<14> Apr 25 16:43:29 host app: message'], + 'regexp/tab' => ['regexp', true, "<14>\tApr 25 16:43:29 host app: message"], + 'string/tab' => ['string', true, "<14>\tApr 25 16:43:29 host app: message"], + 'regexp/two leading spaces' => ['regexp', false, ' Apr 25 16:43:29 host app: message'], + 'string/two leading spaces' => ['string', false, ' Apr 25 16:43:29 host app: message'], + 'regexp/leading tab' => ['regexp', false, "\tApr 25 16:43:29 host app: message"], + 'string/leading tab' => ['string', false, "\tApr 25 16:43:29 host app: message"], + 'regexp/long priority' => ['regexp', true, '<1234> Apr 25 16:43:29 host app: message'], + 'string/long priority' => ['string', true, '<1234> Apr 25 16:43:29 host app: message'], + 'regexp/non-numeric priority' => ['regexp', true, ' Apr 25 16:43:29 host app: message'], + 'string/non-numeric priority' => ['string', true, ' Apr 25 16:43:29 host app: message'], + ) + def test_parse_does_not_broaden_rfc3164_priority_separator(data) + parser_engine, with_priority, text = data + @parser.configure('parser_engine' => parser_engine, 'with_priority' => with_priority) + parsed = false + + begin + @parser.instance.parse(text) do |time, record| + parsed = !!(time && record) + end + rescue Fluent::TimeParser::TimeParseError + # The regexp and string parsers report invalid input differently. + end + + assert_false(parsed) + end + + data( + 'zero' => ['<0> Apr 25 16:43:29 host app: message', 0], + 'two digits with leading zero' => ['<01> Apr 25 16:43:29 host app: message', 1], + 'three digits with leading zeros' => ['<001> Apr 25 16:43:29 host app: message', 1], + ) + def test_string_parser_accepts_spaced_numeric_priority_with_leading_zeros(data) + text, expected_priority = data + @parser.configure('parser_engine' => 'string', 'with_priority' => true) + result = nil + @parser.instance.parse(text) do |time, record| + result = [time, record] + end + + time, record = result + assert_not_nil(time) + assert_equal(expected_priority, record['pri']) + assert_equal('host', record['host']) + assert_equal('app', record['ident']) + assert_equal('message', record['message']) + end + + data( + 'long priority' => '<1234> Apr 25 16:43:29 host app: message', + 'non-numeric priority' => ' Apr 25 16:43:29 host app: message', + 'partially numeric priority' => '<1a> Apr 25 16:43:29 host app: message', + 'non-ASCII priority' => '<é> Apr 25 16:43:29 host app: message', + ) + def test_leading_space_time_format_does_not_broaden_priority_syntax(text) + @parser.configure( + 'parser_engine' => 'string', + 'time_format' => ' %b %d %H:%M:%S', + 'with_priority' => true, + ) + parsed = false + @parser.instance.parse(text) do |time, record| + parsed = !!(time && record) + end + + assert_false(parsed) + end + + data( + 'regexp/rfc5424/with priority' => ['regexp', 'rfc5424', true], + 'string/rfc5424/with priority' => ['string', 'rfc5424', true], + 'regexp/auto/with priority' => ['regexp', 'auto', true], + 'string/auto/with priority' => ['string', 'auto', true], + 'regexp/rfc5424/without priority' => ['regexp', 'rfc5424', false], + 'string/rfc5424/without priority' => ['string', 'rfc5424', false], + 'regexp/auto/without priority' => ['regexp', 'auto', false], + 'string/auto/without priority' => ['string', 'auto', false], + ) + def test_parse_does_not_accept_space_after_rfc5424_priority(data) + parser_engine, message_format, with_priority = data + @parser.configure( + 'parser_engine' => parser_engine, + 'message_format' => message_format, + 'with_priority' => with_priority, + ) + text = with_priority ? '<14> 1 2026-04-25T16:43:29Z host app 123 ID - message' : ' 1 2026-04-25T16:43:29Z host app 123 ID - message' + parsed = false + + begin + @parser.instance.parse(text) do |time, record| + parsed = !!(time && record) + end + rescue Fluent::TimeParser::TimeParseError + # The regexp and string parsers report invalid input differently. + end + + assert_false(parsed) + end + data('regexp' => 'regexp', 'string' => 'string') def test_parse_rfc5452_with_priority(param) @parser.configure('with_priority' => true, 'parser_type' => param, 'message_format' => 'rfc5424')