Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions lib/solid_queue/configuration.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ class Configuration

validate :ensure_configured_processes
validate :ensure_valid_recurring_tasks
validate :ensure_correctly_sized_thread_pool

class Process < Struct.new(:kind, :attributes)
def instantiate
Expand Down Expand Up @@ -69,6 +68,13 @@ def standalone?
mode.fork? || @options[:standalone]
end

def warn_about_undersized_thread_pool
if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads
SolidQueue.logger.warn "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " +
"database connection pool is #{db_pool_size}. Increase it in `config/database.yml`"
end
end

private
attr_reader :options

Expand All @@ -88,13 +94,6 @@ def ensure_valid_recurring_tasks
end
end

def ensure_correctly_sized_thread_pool
if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads
errors.add(:base, "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " +
"database connection pool is #{db_pool_size}. Increase it in `config/database.yml`")
end
end

def default_options
{
mode: ENV["SOLID_QUEUE_SUPERVISOR_MODE"] || :fork,
Expand Down
2 changes: 2 additions & 0 deletions lib/solid_queue/supervisor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ def start(**options)
configuration = Configuration.new(**options)

if configuration.valid?
configuration.warn_about_undersized_thread_pool

klass = configuration.mode.fork? ? ForkSupervisor : AsyncSupervisor
klass.new(configuration).tap(&:start)
else
Expand Down
31 changes: 31 additions & 0 deletions test/unit/async_supervisor_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,42 @@ class AsyncSupervisorTest < ActiveSupport::TestCase
end
end

test "warns on boot when the thread pool is larger than the database connection pool" do
log = StringIO.new
with_solid_queue_logger(ActiveSupport::Logger.new(log)) do
supervisor = run_supervisor_as_thread(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ], dispatchers: [])
wait_for_registered_processes(2, timeout: 3.seconds) # supervisor + 1 worker
ensure
supervisor.stop
end

assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+\. Increase it in `config\/database.yml`/, log.string
end

test "does not warn on boot when the database connection pool is large enough" do
log = StringIO.new
with_solid_queue_logger(ActiveSupport::Logger.new(log)) do
supervisor = run_supervisor_as_thread(workers: [ { queues: "background", threads: 1, polling_interval: 10 } ], dispatchers: [])
wait_for_registered_processes(2, timeout: 3.seconds) # supervisor + 1 worker
ensure
supervisor.stop
end

assert_no_match /the database connection pool is/, log.string
end

private
def run_supervisor_as_thread(**options)
SolidQueue::Supervisor.start(mode: :async, standalone: false, **options.with_defaults(skip_recurring: true))
end

def with_solid_queue_logger(logger)
old_logger, SolidQueue.logger = SolidQueue.logger, logger
yield
ensure
SolidQueue.logger = old_logger
end

def simulate_orphaned_executions(count)
count.times { |i| StoreResultJob.set(queue: :new_queue).perform_later(i) }
process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-123")
Expand Down
23 changes: 19 additions & 4 deletions test/unit/configuration_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -165,14 +165,29 @@ class ConfigurationTest < ActiveSupport::TestCase
assert_not configuration.valid?
assert_equal [ "No processes configured" ], configuration.errors.full_messages

# Not enough DB connections
# Not enough DB connections: still valid so boot is not blocked
configuration = SolidQueue::Configuration.new(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ])
assert_not configuration.valid?
assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+. Increase it in `config\/database.yml`/,
configuration.errors.full_messages.first
assert configuration.valid?
end

test "warns on boot when the database pool is smaller than the thread pool" do
log = StringIO.new
with_solid_queue_logger(ActiveSupport::Logger.new(log)) do
configuration = SolidQueue::Configuration.new(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ])
configuration.warn_about_undersized_thread_pool
end

assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+\. Increase it in `config\/database.yml`/, log.string
end

private
def with_solid_queue_logger(logger)
old_logger, SolidQueue.logger = SolidQueue.logger, logger
yield
ensure
SolidQueue.logger = old_logger
end

def assert_processes(configuration, kind, count, **attributes)
processes = configuration.configured_processes.select { |p| p.kind == kind }
assert_equal count, processes.size
Expand Down
Loading