add tests for invalid keylen guard (issue #31/#33)

.github/workflows/ci.yml

@@ -46,3 +46,18 @@ jobs:
           bundler-cache: true
       - run: bundle exec rake compile
       - run: bundle exec rake test
+
+  memcheck:
+    name: Memory leak check (valgrind)
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_GEMFILE: Gemfile.memcheck
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+      - run: sudo apt-get install -y valgrind
+      - run: bundle exec rake compile
+      - run: bundle exec rake memcheck

Gemfile.memcheck

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'ruby_memcheck'

Rakefile

@@ -7,6 +7,15 @@ require 'rdoc/task'
 require 'benchmark'
 require 'rake_compiler_dock'
 
+begin
+  require 'ruby_memcheck'
+  RubyMemcheck::TestTask.new(memcheck: :compile) do |t|
+    t.libs << 'test'
+    t.test_files = FileList['test/**/*_test.rb']
+  end
+rescue LoadError
+end
+
 CLEAN.add("{ext,lib}/**/*.{o,so}", "pkg")
 
 cross_rubies = ["3.4.0", "3.3.0", "3.2.0", "3.1.0", "3.0.0", "2.7.0"]

test/bcrypt_pnkdf/engine_test.rb

@@ -66,6 +66,21 @@ def test_ruby_and_native_returns_the_same
   end
 
 
+  # Issue #31/33: xmalloc(okeylen) was called before the guards inside
+  # bcrypt_pbkdf(), so out-of-range keylen caused a heap allocation that was
+  # never freed when bcrypt_pbkdf() returned -1. `loop { key("p","s",2000,1) }`
+  # on unfixed code would exhaust memory.
+  def test_invalid_keylen_returns_nil
+    assert_nil BCryptPbkdf::Engine.__bc_crypt_pbkdf("pass", "salt", 0, 1)
+    assert_nil BCryptPbkdf::Engine.__bc_crypt_pbkdf("pass", "salt", 1025, 1)
+    assert_nil BCryptPbkdf::Engine.__bc_crypt_pbkdf("pass", "salt", 2000, 1)
+    assert_nil BCryptPbkdf::Engine.__bc_crypt_pbkdf("pass", "salt", 32, 0)
+  end
+
+  def test_invalid_keylen_does_not_leak_memory
+    1000.times { BCryptPbkdf::Engine.__bc_crypt_pbkdf("pass", "salt", 2000, 1) }
+  end
+
   def table
     [
       ["pass2", "salt2", 12, 2, [214, 14, 48, 162, 131, 206, 121, 176, 50, 104, 231, 252]],