chore(build): Add configurations and scripts for compiling, testing, and packaging

Author: wudidapaopao

.gitignore

@@ -10,12 +10,12 @@ Gemfile.lock
 .vscode
 
 doc/
+deps/
 gems/
 issues/
 pkg/
 ports/
 tmp/
-tmp_chdb/
 vendor/
 test/
 testdb/

Gemfile

@@ -8,7 +8,6 @@ group :test do
   gem 'rake-compiler', '1.2.9'
   gem 'rake-compiler-dock', '1.9.1'
   gem 'rspec', '3.12.0'
-  gem 'ruby_memcheck', '3.0.1' if Gem::Platform.local.os == 'linux'
 end
 
 group :development do

ext/chdb/extconf.rb

@@ -1,20 +1,138 @@
 # frozen_string_literal: true
 
+require 'fileutils'
 require 'mkmf'
+require 'yaml'
 
-system_found = have_library('chdb') && have_header('chdb.h')
+module ChDB
+  module ExtConf
+    class << self
+      def configure
+        configure_cross_compiler
 
-unless system_found
-  abort "chdb.h or chdb library not found! Please install chdb development files.\n" \
-        "You can try installing with: gem install chdb -- --with-opt-dir=/usr/local/lib \n" \
-        'Or any other path that contains chdb.h and libchdb.so'
-end
+        download_and_extract
+
+        configure_extension
+
+        create_makefile('chdb/chdb_native')
+      end
+
+      def configure_cross_compiler
+        RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
+        ENV['CC'] = RbConfig::CONFIG['CC']
+      end
+
+      def libname
+        'chdb'
+      end
+
+      def configure_extension
+        include_path = File.expand_path('ext/chdb/include', package_root_dir)
+        append_cppflags("-I#{include_path}")
+
+        lib_path = File.expand_path('ext/chdb/lib', package_root_dir)
+        append_ldflags("-L#{lib_path}")
+
+        append_ldflags("-Wl,-rpath,'$$ORIGIN/../lib'")
+
+        abort_could_not_find('chdb.h') unless find_header('chdb.h', include_path)
+
+        return if find_library(libname, nil, lib_path)
+
+        abort_could_not_find(libname)
+      end
+
+      def abort_could_not_find(missing)
+        message = <<~MSG
+          Could not find #{missing}.
+          Please visit https://github.com/chdb-io/chdb-ruby for installation instructions.
+        MSG
+        abort("\n#{message}\n")
+      end
+
+      def download_and_extract
+        target_platform = determine_target_platform
+        version = fetch_chdb_version
+        download_dir = setup_download_directory(target_platform, version)
 
-$srcs = Dir['*.c'] + ['chdb.c'] # rubocop:disable Style/GlobalVars
+        unless Dir.exist?(download_dir)
+          file_name = get_file_name(target_platform)
+          url = build_download_url(version, file_name)
+          download_tarball(url, download_dir, file_name)
+          extract_tarball(download_dir, file_name)
+        end
+
+        copy_files(download_dir, version)
+      end
+
+      private
+
+      def determine_target_platform
+        ENV['TARGET'] || host_platform
+      end
+
+      def fetch_chdb_version
+        dependencies = YAML.load_file(File.join(package_root_dir, 'dependencies.yml'), symbolize_names: true)
+        dependencies[:chdb][:version]
+      end
+
+      def setup_download_directory(target_platform, version)
+        download_dir = File.join(package_root_dir, 'deps', version, target_platform)
+        FileUtils.mkdir_p(download_dir)
+        download_dir
+      end
+
+      def get_file_name(target_platform)
+        case target_platform
+        when 'aarch64-linux-gnu' then 'linux-aarch64-libchdb.tar.gz'
+        when 'x86_64-linux-gnu' then 'linux-x86_64-libchdb.tar.gz'
+        when 'arm64-darwin'     then 'macos-arm64-libchdb.tar.gz'
+        when 'x86_64-darwin'    then 'macos-x86_64-libchdb.tar.gz'
+        else raise "Unsupported platform: #{target_platform}"
+        end
+      end
+
+      def build_download_url(version, file_name)
+        "https://github.com/chdb-io/chdb/releases/download/v#{version}/#{file_name}"
+      end
+
+      def download_tarball(url, download_dir, file_name)
+        tarball = File.join(download_dir, file_name)
+        puts "Downloading chdb library for #{determine_target_platform}..."
+        URI.open(url) do |remote| # rubocop:disable Security/Open
+          IO.copy_stream(remote, tarball)
+        end
+      end
+
+      def extract_tarball(download_dir, file_name)
+        tarball = File.join(download_dir, file_name)
+        system("tar xzf #{tarball} -C #{download_dir}")
+      end
+
+      def copy_files(download_dir, _version)
+        ext_chdb_path = File.join(package_root_dir, 'ext/chdb')
+        [%w[include *.h], %w[lib *.so], %w[lib *.dylib]].each do |(src_dir, pattern)|
+          src = File.join(download_dir, src_dir, pattern)
+          dest = File.join(ext_chdb_path, src_dir)
+          FileUtils.mkdir_p(dest)
+          FileUtils.cp_r(Dir.glob(src), dest, remove_destination: true)
+        end
+      end
+
+      def host_platform
+        RbConfig::CONFIG['host_os'].downcase
+      end
+
+      def package_root_dir
+        File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+      end
+    end
+  end
+end
 
-if RbConfig::CONFIG['GCC'] == 'yes'
-  $CFLAGS << ' -Wno-declaration-after-statement' # rubocop:disable Style/GlobalVars
-  $CFLAGS = $CFLAGS.gsub(/-Wno-self-assign|-Wno-parentheses-equality|-Wno-constant-logical-operand/, '') # rubocop:disable Style/GlobalVars
+if arg_config('--download-dependencies')
+  ChDB::ExtConf.download_and_extract
+  exit!(0)
 end
 
-create_makefile('chdb/chdb')
+ChDB::ExtConf.configure

lib/chdb.rb

@@ -2,9 +2,9 @@
 
 begin
   RUBY_VERSION =~ /(\d+\.\d+)/
-  require "ChDB/#{Regexp.last_match(1)}/chdb"
+  require "chdb/#{Regexp.last_match(1)}/chdb_native"
 rescue LoadError
-  require 'ChDB/chdb'
+  require 'chdb/chdb_native'
 end
 
 require 'chdb/database'

rakelib/native.rake

@@ -1,89 +1,30 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rubygems/package_task"
-require "rake/extensiontask"
-require "rake_compiler_dock"
-require "yaml"
-
-cross_platforms = [
-  "aarch64-linux-gnu",
-  "x86_64-linux-gnu",
-  "arm64-darwin",
-  "x86_64-darwin"
+require 'bundler/gem_tasks'
+require 'rubygems/package_task'
+require 'rake/extensiontask'
+require 'rake_compiler_dock'
+require 'yaml'
+
+cross_platforms = %w[
+  aarch64-linux-gnu
+  x86_64-linux-gnu
+  arm64-darwin
+  x86_64-darwin
 ]
 
-RakeCompilerDock.set_ruby_cc_version("~> 3.1")
+RakeCompilerDock.set_ruby_cc_version('~> 3.1')
 
 # Gem::PackageTask.new(CHDB_SPEC).define # packaged_tarball version of the gem for platform=ruby
-task "package" => cross_platforms.map { |p| "gem:#{p}" } # "package" task for all the native platforms
-
-module CHDBDependency
-  class << self
-    def setup
-      dependencies = YAML.load_file(File.join(__dir__, "..", "dependencies.yml"), symbolize_names: true)
-      chdb_info = dependencies[:chdb]
-      version = chdb_info[:version]
-      
-      cross_platforms.each do |platform|
-        platform_key = platform.gsub(/-/, '_').to_sym
-        next unless chdb_info[:platforms][platform_key]
-        
-        download_and_extract(platform, version)
-      end
-    end
-
-    private
-
-    def download_and_extract(platform, version)
-      file_name = case platform
-                  when 'aarch64-linux-gnu' then 'linux-aarch64-libchdb.tar.gz'
-                  when 'x86_64-linux-gnu' then 'linux-x86_64-libchdb.tar.gz'
-                  when 'arm64-darwin'     then 'macos-arm64-libchdb.tar.gz'
-                  when 'x86_64-darwin'    then 'macos-x86_64-libchdb.tar.gz'
-                  end
-
-      url = "https://github.com/chdb-io/chdb/releases/download/v#{version}/#{file_name}"
-    
-      archive_dir = File.join("ports", "archives")
-      FileUtils.mkdir_p(archive_dir)
-      
-      tarball = File.join(archive_dir, name)
-      unless File.exist?(tarball)
-        puts "Downloading #{name}..."
-        URI.open(url) do |remote|
-          IO.copy_stream(remote, tarball)
-        end
-      end
-
-      tmp_dir = File.join(archive_dir, "tmp_chdb")
-      FileUtils.rm_rf(tmp_dir)
-      FileUtils.mkdir_p(tmp_dir)
-      
-      system("tar xzf #{tarball} -C #{tmp_dir}")
-
-      ext_chdb_path = File.expand_path("ext/chdb", __dir__)
-      [%w[include *.h], %w[lib *.so], %w[lib *.dylib]].each do |(src_dir, pattern)|
-        src = File.join(tmp_dir, src_dir, pattern)
-        dest = File.join(ext_chdb_path, src_dir)
-        FileUtils.mkdir_p(dest)
-        FileUtils.cp_r(Dir.glob(src), dest, remove_destination: true)
-      end
-
-      # 清理临时目录
-      FileUtils.rm_rf(tmp_dir)
-    end
-  end
-end
+# 'package' task for all the native platforms
+task 'package' => cross_platforms.map { |p| "gem:#{p}" }
 
 def gem_build_path
-  File.join("pkg", CHDB_SPEC.full_name)
+  File.join('pkg', CHDB_SPEC.full_name)
 end
 
 def add_file_to_gem(relative_source_path)
-  if relative_source_path.nil? || !File.exist?(relative_source_path)
-    raise "Cannot find file '#{relative_source_path}'"
-  end
+  raise "Cannot find file '#{relative_source_path}'" if relative_source_path.nil? || !File.exist?(relative_source_path)
 
   dest_path = File.join(gem_build_path, relative_source_path)
   dest_dir = File.dirname(dest_path)
@@ -96,72 +37,71 @@ def add_file_to_gem(relative_source_path)
 end
 
 task gem_build_path do
-  dependencies = YAML.load_file(File.join(__dir__, "..", "dependencies.yml"), symbolize_names: true)
+  dependencies = YAML.load_file(File.join(__dir__, '..', 'dependencies.yml'), symbolize_names: true)
   sqlite_tarball = File.basename(dependencies[:sqlite3][:files].first[:url])
-  archive = Dir.glob(File.join("ports", "archives", sqlite_tarball)).first
+  archive = Dir.glob(File.join('ports', 'archives', sqlite_tarball)).first
   add_file_to_gem(archive)
 end
 
-Rake::ExtensionTask.new("chdb_native", CHDB_SPEC) do |ext|
-  ext.ext_dir = "ext/chdb"
-  ext.lib_dir = "lib/chdb"
+Rake::ExtensionTask.new('chdb_native', CHDB_SPEC) do |ext|
+  ext.ext_dir = 'ext/chdb'
+  ext.lib_dir = 'lib/chdb'
   ext.cross_compile = true
   ext.cross_platform = cross_platforms
-  ext.cross_config_options << "--enable-cross-build" # so extconf.rb knows we're cross-compiling
-  
-  ext.prerequisites << :download_chdb_deps
+  # so extconf.rb knows we're cross-compiling
+  ext.cross_config_options << '--enable-cross-build'
 end
 
-namespace "gem" do
+namespace 'gem' do
   cross_platforms.each do |platform|
     desc "build native gem for #{platform}"
     task platform do
-      RakeCompilerDock.sh(<<~EOF, platform: platform, verbose: true)
+      RakeCompilerDock.sh(<<~COMMAND_END, platform: platform, verbose: true)
         gem install bundler --no-document &&
         bundle &&
         bundle exec rake gem:#{platform}:buildit
-      EOF
+      COMMAND_END
     end
 
     namespace platform do
       # this runs in the rake-compiler-dock docker container
-      task "buildit" do
+      task 'buildit' do
         # use Task#invoke because the pkg/*gem task is defined at runtime
         Rake::Task["native:#{platform}"].invoke
         Rake::Task["pkg/#{CHDB_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke
       end
     end
   end
 
-  desc "build native gem for all platforms"
-  task "all" => [cross_platforms, "gem"].flatten
+  desc 'build native gem for all platforms'
+  task 'all' => [cross_platforms, 'gem'].flatten
 end
 
-desc "Temporarily set VERSION to a unique timestamp"
-task "set-version-to-timestamp" do
+desc 'Temporarily set VERSION to a unique timestamp'
+task 'set-version-to-timestamp' do
   # this task is used by bin/test-gem-build
   # to test building, packaging, and installing a precompiled gem
   version_constant_re = /^\s*VERSION\s*=\s*["'](.*)["']$/
 
-  version_file_path = File.join(__dir__, "../lib/chdb/version.rb")
+  version_file_path = File.join(__dir__, '../lib/chdb/version.rb')
   version_file_contents = File.read(version_file_path)
 
   current_version_string = version_constant_re.match(version_file_contents)[1]
   current_version = Gem::Version.new(current_version_string)
 
-  fake_version = Gem::Version.new(format("%s.test.%s", current_version.bump, Time.now.strftime("%Y.%m%d.%H%M")))
+  bumped_version = current_version.bump
+  timestamp = Time.now.strftime('%Y.%m%d.%H%M')
+  fake_version = Gem::Version.new(format('%<bumped_version>s.test.%<timestamp>s',
+                                         bumped_version: bumped_version,
+                                         timestamp: timestamp))
 
   unless version_file_contents.gsub!(version_constant_re, "    VERSION = \"#{fake_version}\"")
-    raise("Could not hack the VERSION constant")
+    raise('Could not hack the VERSION constant')
   end
 
   File.write(version_file_path, version_file_contents)
 
   puts "NOTE: wrote version as \"#{fake_version}\""
 end
 
-CLEAN.add("{ext,lib}/**/*.{o,so}", "pkg", "ext/chdb/{include,lib}")
-
-task :download_chdb_deps do
-  CHDBDependency.setup
-end
+CLEAN.add('{ext,lib}/**/*.{o,so}', 'pkg', 'ext/chdb/{include,lib}')

rakelib/test.rake

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'rspec/core/rake_task'
+
+CHDB_SPEC = Bundler.load_gemspec('chdb.gemspec')
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+task test: :spec