Run the RBS parser on JRuby via WebAssembly (JRuby support, step 3)

.github/workflows/jruby.yml

@@ -0,0 +1,65 @@
+name: JRuby
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - ".github/workflows/jruby.yml"
+      - "include/**"
+      - "src/**"
+      - "wasm/**"
+      - "lib/rbs/wasm/**"
+      - "lib/rbs.rb"
+      - "Rakefile"
+
+permissions:
+  contents: read
+
+env:
+  # Keep in sync with .github/workflows/wasm.yml.
+  WASI_SDK_VERSION: "33"
+  WASI_SDK_RELEASE: "33.0"
+
+jobs:
+  test:
+    name: jruby
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+
+      # Build the .wasm and fetch the Chicory jars with CRuby + the WASI SDK,
+      # then run RBS itself on JRuby against those artifacts.
+      - name: Set up Ruby (to assemble the WebAssembly runtime)
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby
+          bundler: none
+      - name: Update rubygems & bundler
+        run: gem update --system
+      - name: Install gems
+        run: |
+          bundle config set --local without libs:profilers
+          bundle install --jobs 4 --retry 3
+      - name: Install the WASI SDK
+        run: |
+          url="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_RELEASE}-x86_64-linux.tar.gz"
+          mkdir -p "$HOME/wasi-sdk"
+          curl -sSL "$url" | tar xz --strip-components=1 -C "$HOME/wasi-sdk"
+          echo "WASI_SDK_PATH=$HOME/wasi-sdk" >> "$GITHUB_ENV"
+      - name: Assemble the JRuby runtime (rbs_parser.wasm + Chicory jars)
+        run: bundle exec rake wasm:jruby_setup
+
+      - name: Set up JRuby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: jruby
+          bundler: none
+      - name: Install runtime and test gems
+        run: gem install prism test-unit --no-document
+      - name: Run RBS's parser on JRuby
+        run: |
+          jruby -Ilib -Itest test/rbs/wasm/jruby_parser_test.rb
+          jruby -Ilib -Itest test/rbs/parser_test.rb

.gitignore

@@ -32,3 +32,7 @@ rust/ruby-rbs/vendor/rbs/
 
 # Compiled WebAssembly module (built by rake wasm:build)
 wasm/*.wasm
+
+# JRuby runtime artifacts (assembled by rake wasm:jruby_setup, bundled in the JRuby gem)
+lib/rbs/wasm/*.wasm
+lib/rbs/wasm/jars/

Rakefile

@@ -609,16 +609,52 @@ namespace :wasm do
   task :check => :build do
     wasmtime = ENV["WASMTIME"] || "wasmtime"
 
-    # `rbs_wasm_selftest` parses a small fixed signature and returns 0 on
+    # `rbs_wasm_selftest` parses a small fixed signature and returns 1 on
     # success. `--invoke` prints the return value to stdout.
     output = IO.popen([wasmtime, "run", "--invoke", "rbs_wasm_selftest", WASM_OUTPUT], err: File::NULL, &:read).to_s.strip
 
-    if output == "0"
+    if output == "1"
       puts "WebAssembly selftest passed."
     else
-      raise "WebAssembly selftest failed: rbs_wasm_selftest returned #{output.inspect} (expected \"0\")"
+      raise "WebAssembly selftest failed: rbs_wasm_selftest returned #{output.inspect} (expected \"1\")"
     end
   end
+
+  # Where the runtime looks for the module and jars by default (see
+  # RBS::WASM::Runtime). These are build artifacts, bundled into the JRuby gem.
+  JRUBY_WASM_DIR = File.expand_path("lib/rbs/wasm", __dir__)
+  CHICORY_VERSION = ENV.fetch("CHICORY_VERSION", "1.7.5")
+  # `compiler` is Chicory's AOT compiler (wasm -> JVM bytecode); the asm* jars
+  # are the ow2 ASM libraries it depends on. Keep ASM_VERSION in sync with what
+  # the pinned Chicory release declares.
+  CHICORY_JARS = %w[wasm runtime log wasi compiler].freeze
+  ASM_VERSION = ENV.fetch("ASM_VERSION", "9.9.1")
+  ASM_JARS = %w[asm asm-tree asm-util asm-commons asm-analysis].freeze
+
+  desc "Download the Chicory and ASM jars the JRuby runtime needs into lib/rbs/wasm/jars"
+  task :vendor_jars do
+    require "open-uri"
+    require "fileutils"
+
+    jars_dir = File.join(JRUBY_WASM_DIR, "jars")
+    FileUtils.mkdir_p(jars_dir)
+
+    downloads = CHICORY_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/com/dylibso/chicory/#{name}/#{CHICORY_VERSION}/#{name}-#{CHICORY_VERSION}.jar"] }
+    downloads += ASM_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/org/ow2/asm/#{name}/#{ASM_VERSION}/#{name}-#{ASM_VERSION}.jar"] }
+
+    downloads.each do |filename, url|
+      puts "Downloading #{url}"
+      URI.open(url) { |io| File.binwrite(File.join(jars_dir, filename), io.read) } # steep:ignore
+    end
+
+    puts "Vendored Chicory #{CHICORY_VERSION} + ASM #{ASM_VERSION} into #{jars_dir}"
+  end
+
+  desc "Assemble everything the JRuby gem needs: the .wasm and the Chicory jars"
+  task :jruby_setup => [:build, :vendor_jars] do
+    cp WASM_OUTPUT, File.join(JRUBY_WASM_DIR, "rbs_parser.wasm")
+    puts "JRuby runtime is ready under #{JRUBY_WASM_DIR}"
+  end
 end
 
 namespace :rust do

Steepfile

@@ -6,6 +6,13 @@ target :lib do
   ignore(
     "lib/rbs/test",
     # "lib/rbs/test.rb"
+
+    # JRuby-only implementations of RBS::Location and RBS::Parser. Like the C
+    # extension, these implement interfaces already described in sig/, and
+    # runtime.rb is Java interop, so they are not type-checked here.
+    "lib/rbs/wasm/location.rb",
+    "lib/rbs/wasm/runtime.rb",
+    "lib/rbs/wasm/parser.rb",
   )
 
   library "pathname", "json", "logger", "monitor", "tsort", "uri", 'dbm', 'pstore', 'singleton', 'shellwords', 'fileutils', 'find', 'digest', 'prettyprint', 'yaml', "psych", "securerandom"

include/rbs/serialize.h

@@ -30,4 +30,10 @@
  */
 rbs_string_t rbs_serialize_node(rbs_allocator_t *allocator, rbs_constant_pool_t *constant_pool, rbs_node_t *node);
 
+/**
+ * Like rbs_serialize_node, but for a bare node list (e.g. the result of
+ * rbs_parse_type_params). Decoded by RBS::WASM::Deserializer.deserialize_node_list.
+ */
+rbs_string_t rbs_serialize_node_list(rbs_allocator_t *allocator, rbs_constant_pool_t *constant_pool, rbs_node_list_t *list);
+
 #endif

lib/rbs.rb

@@ -69,7 +69,14 @@
 require "rbs/type_alias_regularity"
 require "rbs/collection"
 
-require "rbs_extension"
+if RUBY_ENGINE == "jruby"
+  # JRuby cannot load the MRI C extension. Run the parser in WebAssembly and
+  # provide pure-Ruby implementations of RBS::Location and RBS::Parser instead.
+  require "rbs/wasm/location"
+  require "rbs/wasm/parser"
+else
+  require "rbs_extension"
+end
 require "rbs/parser_aux"
 require "rbs/location_aux"
 

lib/rbs/wasm/deserializer.rb

@@ -19,6 +19,18 @@ def self.deserialize(bytes, buffer)
         new(bytes, buffer).read_node
       end
 
+      # Deserialize a bare node list (rbs_serialize_node_list), e.g. the result
+      # of RBS::Parser._parse_type_params.
+      def self.deserialize_node_list(bytes, buffer)
+        new(bytes, buffer).read_node_list
+      end
+
+      # Deserialize the token stream produced by rbs_wasm_lex into the
+      # [type, location] pairs RBS::Parser._lex returns.
+      def self.deserialize_tokens(bytes, buffer)
+        new(bytes, buffer).read_tokens
+      end
+
       def initialize(bytes, buffer)
         @bytes = bytes
         @buffer = buffer
@@ -50,6 +62,23 @@ def read_node
         end
       end
 
+      def read_node_list
+        Array.new(read_count) { read_node }
+      end
+
+      # The lex stream has no leading count: read records until the buffer is
+      # exhausted. Each is a token type name followed by its character range.
+      def read_tokens
+        tokens = [] #: Array[[ Symbol, Location ]]
+        until @pos >= @bytes.bytesize
+          type = read_string(Encoding::UTF_8).to_sym
+          start_char = read_i32
+          end_char = read_i32
+          tokens << [type, RBS::Location.new(@buffer, start_char, end_char)]
+        end
+        tokens
+      end
+
       private
 
       def read_struct(entry)
@@ -87,10 +116,6 @@ def read_field(reader)
         end
       end
 
-      def read_node_list
-        Array.new(read_count) { read_node }
-      end
-
       def read_hash
         hash = {} #: Hash[untyped, untyped]
         read_count.times do

lib/rbs/wasm/location.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RBS
+  # Pure-Ruby implementation of the primitives that back RBS::Location.
+  #
+  # On CRuby these come from the C extension (ext/rbs_extension/legacy_location.c).
+  # JRuby loads this instead, before rbs/location_aux.rb layers the public API on
+  # top, so RBS::Location behaves identically without the native extension.
+  class Location
+    attr_reader :buffer
+
+    def initialize(buffer, start_pos, end_pos)
+      @buffer = buffer
+      @start_pos = start_pos
+      @end_pos = end_pos
+      @required_children = {} #: Hash[Symbol, [ Integer, Integer ]]
+      @optional_children = {} #: Hash[Symbol, [ Integer, Integer ]?]
+    end
+
+    def _start_pos
+      @start_pos
+    end
+
+    def _end_pos
+      @end_pos
+    end
+
+    def _add_required_child(name, start_pos, end_pos)
+      @required_children[name] = [start_pos, end_pos]
+    end
+
+    def _add_optional_child(name, start_pos, end_pos)
+      @optional_children[name] = [start_pos, end_pos]
+    end
+
+    def _add_optional_no_child(name)
+      @optional_children[name] = nil
+    end
+
+    def _required_keys
+      @required_children.keys
+    end
+
+    def _optional_keys
+      @optional_children.keys
+    end
+
+    def [](name)
+      if (range = @required_children[name])
+        return Location.new(@buffer, range[0], range[1])
+      end
+
+      if @optional_children.key?(name)
+        range = @optional_children[name]
+        return range && Location.new(@buffer, range[0], range[1])
+      end
+
+      nil
+    end
+  end
+end