Use String#undump instead of eval

lib/racc/grammarfileparser.rb

@@ -401,6 +401,7 @@ def add_user_code(label, src)
   class GrammarFileScanner
 
     def initialize(str, filename = '-')
+      @encoding = str.encoding
       @lines  = str.b.split(/\n|\r\n|\r/)
       @filename = filename
       @lineno = -1
@@ -456,7 +457,24 @@ def yylex0
           elsif ch = reads(/\A./)
             case ch
             when '"', "'"
-              yield [:STRING, eval(scan_quoted(ch))]
+              string_literal = scan_quoted(ch)
+              if ch == "'"
+                # We can't use String#undump for '...'.
+                string = string_literal[1..-2].gsub(/\\\\|\\'/) do |matched|
+                  matched[1]
+                end
+              else
+                # String#undump rejects non-ASCII
+                # characters. string_literal is ASCII-8BIT because
+                # @lines is ASCII-8BIT. We can use \xHH here to
+                # convert non-ASCII characters to ASCII characters.
+                string_literal = string_literal.gsub(/[\x80-\xff]/n) do |c|
+                  "\\x%02x" % c.ord
+                end
+                string = string_literal.undump
+              end
+              string.force_encoding(@encoding)
+              yield [:STRING, string]
             when '{'
               lineno = lineno()
               yield [:ACTION, SourceText.new(scan_action(), @filename, lineno)]

test/test_grammar_file_parser.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 require File.expand_path(File.join(__dir__, 'case'))
 
 module Racc
@@ -68,5 +70,48 @@ class Parse
 
       assert_equal "4: terminal and nonterminal symbols cannot start with ':', but got :TERM2", error.message
     end
+
+    def test_non_ascii_string
+      parser = Racc::GrammarFileParser.new
+
+      result = parser.parse(<<~RACC, 'non_ascii.y')
+        class Parse
+        rule
+          target : "あ" 'い' "\\u3046" "\\n"
+        end
+      RACC
+
+      strings = result.grammar.symbols.map(&:value).grep(String)
+      assert_equal ["あ", "い", "う", "\n"], strings
+    end
+
+    def test_non_ascii_string_in_non_utf8_source
+      parser = Racc::GrammarFileParser.new
+
+      result = parser.parse(<<~RACC.encode(Encoding::EUC_JP), 'non_ascii_euc.y')
+        class Parse
+        rule
+          target : "あ"
+        end
+      RACC
+
+      strings = result.grammar.symbols.map(&:value).grep(String)
+      assert_equal ["あ".encode(Encoding::EUC_JP)], strings
+    end
+
+    def test_non_ascii_string_interned_consistently
+      parser = Racc::GrammarFileParser.new
+
+      result = parser.parse(<<~RACC, 'non_ascii_intern.y')
+        class Parse
+        rule
+          target : "あ" other
+          other : 'あ'
+        end
+      RACC
+
+      strings = result.grammar.symbols.map(&:value).grep(String)
+      assert_equal ["あ"], strings
+    end
   end
 end