From 8afacf7b29ad3f6ea2752b9bab301097ea3eeabc Mon Sep 17 00:00:00 2001 From: Ajay Krishnan Date: Wed, 24 Jun 2026 00:04:40 -0700 Subject: [PATCH] Warn instead of raising when the connection pool is smaller than the thread pool Solid Queue refused to boot when the Active Record connection pool was smaller than the worker thread pool. This blocked legitimate setups, such as I/O-bound queues that run many threads against a deliberately small pool. Per the discussion in #736, downgrade the check from a validation error to a SolidQueue.logger warning emitted once on the boot path in Supervisor.start, so undersized configurations boot while still surfacing the advisory. valid? stays purely about whether we can boot. Closes #736 --- lib/solid_queue/configuration.rb | 15 +++++++-------- lib/solid_queue/supervisor.rb | 2 ++ test/unit/async_supervisor_test.rb | 31 ++++++++++++++++++++++++++++++ test/unit/configuration_test.rb | 23 ++++++++++++++++++---- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index e63a000ca..300bfd100 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -6,7 +6,6 @@ class Configuration validate :ensure_configured_processes validate :ensure_valid_recurring_tasks - validate :ensure_correctly_sized_thread_pool class Process < Struct.new(:kind, :attributes) def instantiate @@ -69,6 +68,13 @@ def standalone? mode.fork? || @options[:standalone] end + def warn_about_undersized_thread_pool + if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads + SolidQueue.logger.warn "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " + + "database connection pool is #{db_pool_size}. Increase it in `config/database.yml`" + end + end + private attr_reader :options @@ -88,13 +94,6 @@ def ensure_valid_recurring_tasks end end - def ensure_correctly_sized_thread_pool - if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads - errors.add(:base, "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " + - "database connection pool is #{db_pool_size}. Increase it in `config/database.yml`") - end - end - def default_options { mode: ENV["SOLID_QUEUE_SUPERVISOR_MODE"] || :fork, diff --git a/lib/solid_queue/supervisor.rb b/lib/solid_queue/supervisor.rb index ae17ec95c..695a066a4 100644 --- a/lib/solid_queue/supervisor.rb +++ b/lib/solid_queue/supervisor.rb @@ -13,6 +13,8 @@ def start(**options) configuration = Configuration.new(**options) if configuration.valid? + configuration.warn_about_undersized_thread_pool + klass = configuration.mode.fork? ? ForkSupervisor : AsyncSupervisor klass.new(configuration).tap(&:start) else diff --git a/test/unit/async_supervisor_test.rb b/test/unit/async_supervisor_test.rb index 6a0f3553b..d8843089a 100644 --- a/test/unit/async_supervisor_test.rb +++ b/test/unit/async_supervisor_test.rb @@ -84,11 +84,42 @@ class AsyncSupervisorTest < ActiveSupport::TestCase end end + test "warns on boot when the thread pool is larger than the database connection pool" do + log = StringIO.new + with_solid_queue_logger(ActiveSupport::Logger.new(log)) do + supervisor = run_supervisor_as_thread(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ], dispatchers: []) + wait_for_registered_processes(2, timeout: 3.seconds) # supervisor + 1 worker + ensure + supervisor.stop + end + + assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+\. Increase it in `config\/database.yml`/, log.string + end + + test "does not warn on boot when the database connection pool is large enough" do + log = StringIO.new + with_solid_queue_logger(ActiveSupport::Logger.new(log)) do + supervisor = run_supervisor_as_thread(workers: [ { queues: "background", threads: 1, polling_interval: 10 } ], dispatchers: []) + wait_for_registered_processes(2, timeout: 3.seconds) # supervisor + 1 worker + ensure + supervisor.stop + end + + assert_no_match /the database connection pool is/, log.string + end + private def run_supervisor_as_thread(**options) SolidQueue::Supervisor.start(mode: :async, standalone: false, **options.with_defaults(skip_recurring: true)) end + def with_solid_queue_logger(logger) + old_logger, SolidQueue.logger = SolidQueue.logger, logger + yield + ensure + SolidQueue.logger = old_logger + end + def simulate_orphaned_executions(count) count.times { |i| StoreResultJob.set(queue: :new_queue).perform_later(i) } process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-123") diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 34f69658b..bf2c6b8ec 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -165,14 +165,29 @@ class ConfigurationTest < ActiveSupport::TestCase assert_not configuration.valid? assert_equal [ "No processes configured" ], configuration.errors.full_messages - # Not enough DB connections + # Not enough DB connections: still valid so boot is not blocked configuration = SolidQueue::Configuration.new(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ]) - assert_not configuration.valid? - assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+. Increase it in `config\/database.yml`/, - configuration.errors.full_messages.first + assert configuration.valid? + end + + test "warns on boot when the database pool is smaller than the thread pool" do + log = StringIO.new + with_solid_queue_logger(ActiveSupport::Logger.new(log)) do + configuration = SolidQueue::Configuration.new(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ]) + configuration.warn_about_undersized_thread_pool + end + + assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+\. Increase it in `config\/database.yml`/, log.string end private + def with_solid_queue_logger(logger) + old_logger, SolidQueue.logger = SolidQueue.logger, logger + yield + ensure + SolidQueue.logger = old_logger + end + def assert_processes(configuration, kind, count, **attributes) processes = configuration.configured_processes.select { |p| p.kind == kind } assert_equal count, processes.size