RUBY-2446 Fix intermittent SDAM test failures (#2132)

* allow some ismasters to not complete

* also allow regular ismasters to be not responded to

* allow in-progress heartbeats on pre-4.4 servers

* fix the ranges

* stop server monitors to avoid races

* fix the check again

* increase heartbeat interval

Co-authored-by: Oleg Pudeyev <oleg@bsdpower.com>

Author: p-mongo

spec/integration/sdam_error_handling_spec.rb

@@ -19,6 +19,7 @@
     new_local_client(SpecConfig.instance.addresses,
       SpecConfig.instance.all_test_options.merge(
         socket_timeout: 3, connect_timeout: 3,
+        heartbeat_frequency: 100,
         # Uncomment to print all events to stdout:
         #sdam_proc: Utils.subscribe_all_sdam_proc(diagnostic_subscriber),
         **Utils.disable_retries_client_options)
@@ -28,6 +29,14 @@
   let(:server) { client.cluster.next_primary }
 
   shared_examples_for 'marks server unknown' do
+    before do
+      server.monitor.stop!
+    end
+
+    after do
+      client.close
+    end
+
     it 'marks server unknown' do
       expect(server).not_to be_unknown
       RSpec::Mocks.with_temporary_scope do
@@ -38,6 +47,14 @@
   end
 
   shared_examples_for 'does not mark server unknown' do
+    before do
+      server.monitor.stop!
+    end
+
+    after do
+      client.close
+    end
+
     it 'does not mark server unknown' do
       expect(server).not_to be_unknown
       RSpec::Mocks.with_temporary_scope do

spec/integration/sdam_events_spec.rb

@@ -69,7 +69,8 @@
         succeeded_events = subscriber.select_succeeded_events(Mongo::Monitoring::Event::ServerHeartbeatSucceeded)
         # Since we gracefully close the client, we expect each heartbeat
         # to complete.
-        started_events.length.should == succeeded_events.length
+        started_events.length.should > 1
+        (succeeded_events.length-1..succeeded_events.length).should include(started_events.length)
       end
     end
 
@@ -105,10 +106,12 @@
         (succeeded_awaited = events.select(&:awaited?)).should_not be_empty
         (succeeded_regular = events.reject(&:awaited?)).should_not be_empty
 
-        # Since we gracefully close the client, we expect each heartbeat
-        # to complete.
-        started_awaited.length.should == succeeded_awaited.length
-        started_regular.length.should == succeeded_regular.length
+        # There may be in-flight ismasters that don't complete, both
+        # regular and awaited.
+        started_awaited.length.should > 1
+        (succeeded_awaited.length-1..succeeded_awaited.length).should include(started_awaited.length)
+        started_regular.length.should > 1
+        (succeeded_regular.length-1..succeeded_regular.length).should include(started_regular.length)
       end
     end
   end