From 01677d06a705dfcc917c32de7374030a10a2606b Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Thu, 7 May 2026 12:12:26 +0200 Subject: [PATCH] Don't create tokens by hand There is some awkward code that dances around the fact that the tokens for a method actually contain a 3 extra tokens that don't exist in the source code. Now `RipperStateLex` is only referenced to actually parse, rest is kept internal --- lib/rdoc/generator/markup.rb | 62 +++++++++--------- lib/rdoc/parser/ruby.rb | 16 +---- test/rdoc/code_object/any_method_test.rb | 42 +----------- test/rdoc/parser/c_test.rb | 27 ++++++++ test/rdoc/parser/ruby_test.rb | 83 ++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 84 deletions(-) diff --git a/lib/rdoc/generator/markup.rb b/lib/rdoc/generator/markup.rb index dc4556c019..58998659e9 100644 --- a/lib/rdoc/generator/markup.rb +++ b/lib/rdoc/generator/markup.rb @@ -86,57 +86,59 @@ class RDoc::CodeObject class RDoc::MethodAttr ## - # Prepend +src+ with line numbers. Relies on the first line of a source - # code listing having: - # - # # File xxxxx, line dddd - # - # If it has this comment then line numbers are added to +src+ and the , - # line dddd portion of the comment is removed. + # Prepend +src+ with line numbers. def add_line_numbers(src) - return unless src.sub!(/\A(.*)(, line (\d+))/, '\1') - first = $3.to_i - 1 - last = first + src.count("\n") - size = last.to_s.length + start_line = line or return + end_line = start_line + src.count("\n") + number_digits = end_line.to_s.length - line = first + current_line = start_line src.gsub!(/^/) do - res = if line == first then - " " * (size + 1) - else - "%2$*1$d " % [size, line] - end + res = "#{current_line.to_s.rjust(number_digits)} " - line += 1 + current_line += 1 res end end + ## + # Prepend +src+ with a comment that declares its location in the source. + + def add_location_comment(src) + path = CGI.escapeHTML(file.relative_name) + if options.line_numbers + src.prepend("# File #{path}\n") + else + src.prepend("# File #{path}, line #{line}\n") + end + end + ## # Turns the method's token stream into HTML. # # Prepends line numbers if +options.line_numbers+ is true. def markup_code - return '' unless @token_stream + return '' if !@token_stream src = RDoc::TokenStream.to_html @token_stream + # add initial whitespace so that the indent gets calculated correctly + src.prepend(' ' * @token_stream.first[:char_no]) if source_language == 'ruby' && @token_stream.first + # dedent the source - indent = src.length - lines = src.lines.to_a - lines.shift if src =~ /\A.*#\ *File/i # remove '# File' comment - lines.each do |line| - if line =~ /^ *(?=\S)/ - n = $~.end(0) - indent = n if n < indent - break if n == 0 - end + common_indent = src.length + src.scan(/^ *(?=\S)/) do |whitespace| + common_indent = whitespace.length if whitespace.length < common_indent + break if common_indent == 0 end - src.gsub!(/^#{' ' * indent}/, '') if indent > 0 + src.gsub!(/^#{' ' * common_indent}/, '') if common_indent > 0 - add_line_numbers(src) if options.line_numbers + if source_language == 'ruby' + add_line_numbers(src) if options.line_numbers + add_location_comment(src) + end src end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 4adaa8ad9e..0a0f690bac 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -314,7 +314,7 @@ def parse_comment_tomdoc(container, comment, line_no, start_line) meth.start_collecting_tokens(:ruby) node = @line_nodes[line_no] - tokens = node ? visible_tokens_from_location(node.location) : [file_line_comment_token(start_line)] + tokens = node ? visible_tokens_from_location(node.location) : [] tokens.each { |token| meth.token_stream << token } container.add_method meth @@ -385,7 +385,7 @@ def handle_meta_method_comment(comment, directives, node) tokens = visible_tokens_from_location(node.location) line_no = node.location.start_line else - tokens = [file_line_comment_token(line_no)] + tokens = [] end internal_add_method( method_name, @@ -498,23 +498,13 @@ def slice_tokens(start_pos, end_pos) # :nodoc: tokens end - def file_line_comment_token(line_no) # :nodoc: - position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no - 1, 0, :on_comment) - position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" - position_comment - end - # Returns tokens from the given location def visible_tokens_from_location(location) - position_comment = file_line_comment_token(location.start_line) - newline_token = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n") - indent_token = RDoc::Parser::RipperStateLex::Token.new(location.start_line, 0, :on_sp, ' ' * location.start_character_column) - tokens = slice_tokens( + slice_tokens( [location.start_line, location.start_character_column], [location.end_line, location.end_character_column] ) - [position_comment, newline_token, indent_token, *tokens] end # Handles `public :foo, :bar` `private :foo, :bar` and `protected :foo, :bar` diff --git a/test/rdoc/code_object/any_method_test.rb b/test/rdoc/code_object/any_method_test.rb index 43dc679d95..b7d9c853ee 100644 --- a/test/rdoc/code_object/any_method_test.rb +++ b/test/rdoc/code_object/any_method_test.rb @@ -148,51 +148,11 @@ def test_call_seq_returns_nil_if_alias_is_missing_from_call_seq assert_nil(alias_to_method.call_seq) end - def test_markup_code - tokens = [ - { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' }, - ] - - @c2_a.collect_tokens(:ruby) - @c2_a.add_tokens(tokens) - - expected = 'CONSTANT' - - assert_equal expected, @c2_a.markup_code - end - - def test_markup_code_with_line_numbers - position_comment = "# File #{@file_name}, line 1" - tokens = [ - { :line_no => 1, :char_no => 0, :kind => :on_comment, :text => position_comment }, - { :line_no => 1, :char_no => position_comment.size, :kind => :on_nl, :text => "\n" }, - { :line_no => 2, :char_no => 0, :kind => :on_const, :text => 'A' }, - { :line_no => 2, :char_no => 1, :kind => :on_nl, :text => "\n" }, - { :line_no => 3, :char_no => 0, :kind => :on_const, :text => 'B' } - ] - - @c2_a.collect_tokens(:ruby) - @c2_a.add_tokens(tokens) - - assert_equal <<-EXPECTED.chomp, @c2_a.markup_code -# File xref_data.rb, line 1 -A -B - EXPECTED - - @c2_a.options.line_numbers = true - assert_equal <<-EXPECTED.chomp, @c2_a.markup_code - # File xref_data.rb -1 A -2 B - EXPECTED - end - def test_markup_code_empty assert_equal '', @c2_a.markup_code end - def test_markup_code_with_variable_expansion + def test_param_seq_with_variable_expansion m = RDoc::AnyMethod.new nil, 'method' m.parent = @c1 m.block_params = '"Hello, #{world}", yield_arg' diff --git a/test/rdoc/parser/c_test.rb b/test/rdoc/parser/c_test.rb index 00b57d5d7d..833c932335 100644 --- a/test/rdoc/parser/c_test.rb +++ b/test/rdoc/parser/c_test.rb @@ -2215,6 +2215,33 @@ def test_markup_format_override assert_equal("markdown", klass.attributes.find {|a| a.name == "default_format"}.comment.format) end + def test_markup_code + # Should not generate line numbers + @top_level.store.options.line_numbers = true + parser = util_parser <<~C + static VALUE + rb_hash_has_value(VALUE hash, VALUE val) { + return Qtrue; + } + + Init_Hash(void) + { + rb_define_method(rb_cHash, "value?", rb_hash_has_value, 1); + } + C + parser.scan + + hash = @store.classes_hash['Hash'] + value_method = hash.method_list.find { |m| m.name == 'value?' } + + assert_equal(<<~EXPECTED.chomp, value_method.markup_code) + static VALUE + rb_hash_has_value(VALUE hash, VALUE val) { + return Qtrue; + } + EXPECTED + end + def test_clear_file_contributions_removes_c_methods content = <<~C /* Document-class: Foo */ diff --git a/test/rdoc/parser/ruby_test.rb b/test/rdoc/parser/ruby_test.rb index 9ec4722b5a..f6843af155 100644 --- a/test/rdoc/parser/ruby_test.rb +++ b/test/rdoc/parser/ruby_test.rb @@ -2529,6 +2529,89 @@ def m2; end assert_equal "ARGF.readlines(a)\nARGF.readlines(b)\nARGF.readlines(c)\nARGF.readlines(d)", m2.call_seq.chomp end + def test_markup_code + util_parser <<~RUBY + class Foo + def bar + end + end + RUBY + + m1, = @top_level.classes.first.method_list + + assert_equal <<~EXPECTED.chomp, m1.markup_code + # File #{@filename}, line 2 + def bar + end + EXPECTED + end + + def test_markup_code_with_line_numbers + @top_level.store.options.line_numbers = true + util_parser <<~RUBY + class Foo + def bar + end + end + RUBY + + m1, = @top_level.classes.first.method_list + + assert_equal <<~EXPECTED.chomp, m1.markup_code + # File #{@filename} + 2 def bar + 3 end + EXPECTED + end + + def test_markup_code_dedent + util_parser <<~RUBY + class Foo + def bar + end + + private + def baz + end + end + RUBY + m1, m2 = @top_level.classes.first.method_list + + assert_equal(<<~EXPECTED.chomp, m1.markup_code) + # File #{@filename}, line 2 + def bar + end + EXPECTED + assert_equal(<<~EXPECTED.chomp, m2.markup_code) + # File #{@filename}, line 6 + def baz + end + EXPECTED + end + + def test_markup_code_empty + util_parser <<~RUBY + class Foo + ## + # :method: ghost_method + + ## + # :method: + # :call-seq: ghost_method2() -> Integer + end + RUBY + + m1, m2 = @top_level.classes.first.method_list + assert_equal( + "# File #{@filename}, line 3", + m1.markup_code.chomp + ) + assert_equal( + "# File #{@filename}, line 6", + m2.markup_code.chomp + ) + end + def util_parser(content) @parser = RDoc::Parser::Ruby.new @top_level, content, @options, @stats @parser.scan