RUBY-1954 Integration test for exceptions referencing second attempt's server for retryable operations (#1551)

* RUBY-1954 Integration test for exceptions referencing second attempt's server for retryable operations

* Also verify servers are reset to unknown

* Require 4.0+

Author: p-mongo

spec/integration/retryable_errors_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+describe 'Retryable writes tests' do
+  # Requirement for fail point
+  min_server_fcv '4.0'
+
+  let(:client) do
+    subscribed_client
+  end
+
+  let(:collection) do
+    client['retryable-writes-error-spec']
+  end
+
+  context 'when retry fails' do
+    require_topology :replica_set
+
+    let(:fail_point_command) do
+      {
+        configureFailPoint: 'failCommand',
+        mode: {times: 1},
+        data: {
+          failCommands: ['find'],
+          errorCode: 11600,
+        },
+      }
+    end
+
+    let(:clear_fail_point_command) do
+      {
+        configureFailPoint: 'failCommand',
+        mode: 'off',
+      }
+    end
+
+    after do
+      ClusterTools.instance.direct_client_for_each_server do |client|
+        client.use(:admin).database.command(clear_fail_point_command)
+      end
+    end
+
+    let(:collection) do
+      client['retryable-writes-error-spec', read: {mode: :secondary_preferred}]
+    end
+
+    let(:events) do
+      events = EventSubscriber.command_started_events('find')
+    end
+
+    let(:first_server) do
+      client.cluster.servers_list.detect do |server|
+        server.address.seed == events.first.address.seed
+      end
+    end
+
+    let(:second_server) do
+      client.cluster.servers_list.detect do |server|
+        server.address.seed == events.last.address.seed
+      end
+    end
+
+    let(:perform_read) do
+      client.cluster.servers_list.each do |server|
+        server.monitor.stop!
+      end
+
+      ClusterTools.instance.direct_client_for_each_server do |client|
+        client.use(:admin).database.command(fail_point_command)
+      end
+
+      begin
+        collection.find(a: 1).to_a
+      rescue Mongo::Error::OperationFailure => @exception
+      else
+        fail('Expected operation to fail')
+      end
+
+      puts @exception.message
+
+      expect(events.length).to eq(2)
+      expect(events.first.address.seed).not_to eq(events.last.address.seed)
+    end
+
+    it 'is reported on the server of the second attempt' do
+      perform_read
+
+      expect(@exception.message).not_to include(first_server.address.seed)
+      expect(@exception.message).to include(second_server.address.seed)
+    end
+
+    it 'marks servers used in both attempts unknown' do
+      perform_read
+
+      expect(first_server).to be_unknown
+
+      expect(second_server).to be_unknown
+    end
+  end
+end

spec/integration/retryable_writes_errors_spec.rb

@@ -1,7 +1,16 @@
 require 'spec_helper'
 
 describe 'Retryable writes errors tests' do
-  describe 'when the storage engine does not support retryable writes but the server does' do
+
+  let(:client) do
+    authorized_client.with(retry_writes: true)
+  end
+
+  let(:collection) do
+    client['retryable-writes-error-spec']
+  end
+
+  context 'when the storage engine does not support retryable writes but the server does' do
     require_mmapv1
     min_server_fcv '3.6'
     require_topology :replica_set, :sharded
@@ -10,14 +19,6 @@
       collection.delete_many
     end
 
-    let(:collection) do
-      client[authorized_collection.name]
-    end
-
-    let(:client) do
-      authorized_client.with(retry_writes: true)
-    end
-
     context 'when a retryable write is attempted' do
       it 'raises an actionable error message' do
         expect {
@@ -27,4 +28,4 @@
       end
     end
   end
-end
+end

spec/support/cluster_tools.rb

@@ -349,8 +349,6 @@ def close_clients
     end
   end
 
-  private
-
   def each_server(&block)
     admin_client.cluster.servers_list.each(&block)
   end
@@ -361,6 +359,8 @@ def direct_client_for_each_server(&block)
     end
   end
 
+  private
+
   def reset_server_states
     each_server do |server|
       server.unknown!

spec/support/event_subscriber.rb

@@ -44,6 +44,13 @@ def started(event)
       end
     end
 
+    # Filters command started events for the specified command name.
+    def command_started_events(command_name)
+      started_events.select do |event|
+        event.command[command_name]
+      end
+    end
+
     # Locates command stated events for the specified command name,
     # asserts that there is exactly one such event, and returns it.
     def single_command_started_event(command_name)