diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts
index d4abb13de0..cbc1c45b42 100644
--- a/src/operations/execute_operation.ts
+++ b/src/operations/execute_operation.ts
@@ -19,6 +19,7 @@ import {
 } from '../error';
 import type { MongoClient } from '../mongo_client';
 import { ReadPreference } from '../read_preference';
+import { TopologyType } from '../sdam/common';
 import {
   DeprioritizedServers,
   sameServerSelector,
@@ -342,7 +343,12 @@ async function executeOperationWithRetries<
         session.unpin({ force: true, forceClear: true });
       }
 
-      deprioritizedServers.add(server.description);
+      if (
+        topology.description.type === TopologyType.Sharded ||
+        operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)
+      ) {
+        deprioritizedServers.add(server.description);
+      }
 
       server = await topology.selectServer(selector, {
         session,
diff --git a/test/integration/retryable-reads/retryable_reads.spec.prose.test.ts b/test/integration/retryable-reads/retryable_reads.spec.prose.test.ts
index ca0576d218..ac63ad2470 100644
--- a/test/integration/retryable-reads/retryable_reads.spec.prose.test.ts
+++ b/test/integration/retryable-reads/retryable_reads.spec.prose.test.ts
@@ -1,7 +1,13 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 import { expect } from 'chai';
 
-import { type Collection, type MongoClient } from '../../mongodb';
+import {
+  type Collection,
+  type CommandFailedEvent,
+  type CommandSucceededEvent,
+  type MongoClient
+} from '../../mongodb';
+import { filterForCommands } from '../shared';
 
 describe('Retryable Reads Spec Prose', () => {
   let client: MongoClient, failPointName;
@@ -136,4 +142,141 @@ describe('Retryable Reads Spec Prose', () => {
       }
     });
   });
+
+  describe('Retrying Reads in a Replica Set', () => {
+    // These tests verify that server deprioritization on replica sets only occurs
+    // for SystemOverloadedError errors.
+
+    const TEST_METADATA: MongoDBMetadataUI = {
+      requires: { mongodb: '>=4.4', topology: 'replicaset' }
+    };
+
+    describe('Retryable Reads Caused by Overload Errors Are Retried on a Different Server', () => {
+      let client: MongoClient;
+      const commandFailedEvents: CommandFailedEvent[] = [];
+      const commandSucceededEvents: CommandSucceededEvent[] = [];
+
+      beforeEach(async function () {
+        // 1. Create a client `client` with `retryReads=true`, `readPreference=primaryPreferred`, and command event monitoring
+        //     enabled.
+        client = this.configuration.newClient({
+          retryReads: true,
+          readPreference: 'primaryPreferred',
+          monitorCommands: true
+        });
+
+        client.on('commandFailed', filterForCommands('find', commandFailedEvents));
+        client.on('commandSucceeded', filterForCommands('find', commandSucceededEvents));
+
+        await client.connect();
+
+        /*
+        * 2. Configure the following fail point for `client`:
+            {
+                configureFailPoint: "failCommand",
+                mode: { times: 1 },
+                data: {
+                    failCommands: ["find"],
+                    errorLabels: ["RetryableError", "SystemOverloadedError"]
+                    errorCode: 6
+                }
+            }
+        * */
+        await client.db('admin').command({
+          configureFailPoint: 'failCommand',
+          mode: { times: 1 },
+          data: {
+            failCommands: ['find'],
+            errorCode: 6,
+            errorLabels: ['RetryableError', 'SystemOverloadedError']
+          }
+        });
+
+        // 3. Reset the command event monitor to clear the failpoint command from its stored events.
+        commandFailedEvents.length = 0;
+        commandSucceededEvents.length = 0;
+      });
+
+      afterEach(async function () {
+        await client?.db('admin').command({ configureFailPoint: 'failCommand', mode: 'off' });
+        await client?.close();
+      });
+
+      it('retries on a different server when SystemOverloadedError', TEST_METADATA, async () => {
+        // 4. Execute a `find` command with `client`.
+        await client.db('test').collection('test').find().toArray();
+
+        // 5. Assert that one failed command event and one successful command event occurred.
+        expect(commandFailedEvents).to.have.lengthOf(1);
+        expect(commandSucceededEvents).to.have.lengthOf(1);
+
+        // 6. Assert that both events occurred on different servers.
+        expect(commandFailedEvents[0].address).to.not.equal(commandSucceededEvents[0].address);
+      });
+    });
+
+    describe('Retryable Reads Caused by Non-Overload Errors Are Retried on the Same Server', () => {
+      let client: MongoClient;
+      const commandFailedEvents: CommandFailedEvent[] = [];
+      const commandSucceededEvents: CommandSucceededEvent[] = [];
+
+      beforeEach(async function () {
+        // 1. Create a client `client` with `retryReads=true`, `readPreference=primaryPreferred`, and command event monitoring
+        //     enabled.
+        client = this.configuration.newClient({
+          retryReads: true,
+          readPreference: 'primaryPreferred',
+          monitorCommands: true
+        });
+
+        client.on('commandFailed', filterForCommands('find', commandFailedEvents));
+        client.on('commandSucceeded', filterForCommands('find', commandSucceededEvents));
+
+        await client.connect();
+
+        /*
+        * 2. Configure the following fail point for `client`:
+            {
+                configureFailPoint: "failCommand",
+                mode: { times: 1 },
+                data: {
+                    failCommands: ["find"],
+                    errorLabels: ["RetryableError"]
+                    errorCode: 6
+                }
+            }
+        * */
+        await client.db('admin').command({
+          configureFailPoint: 'failCommand',
+          mode: { times: 1 },
+          data: {
+            failCommands: ['find'],
+            errorCode: 6,
+            errorLabels: ['RetryableError']
+          }
+        });
+
+        // 3. Reset the command event monitor to clear the failpoint command from its stored events.
+        commandFailedEvents.length = 0;
+        commandSucceededEvents.length = 0;
+      });
+
+      afterEach(async function () {
+        await client?.db('admin').command({ configureFailPoint: 'failCommand', mode: 'off' });
+        await client?.close();
+      });
+
+      it('retries on the same server when no SystemOverloadedError', TEST_METADATA, async () => {
+        // 4. Execute a `find` command with `client`.
+        await client.db('test').collection('test').find().toArray();
+
+        // 5. Assert that one failed command event and one successful command event occurred.
+        expect(commandFailedEvents).to.have.lengthOf(1);
+        expect(commandSucceededEvents).to.have.lengthOf(1);
+
+        // 6. Assert that both events occurred on the same server.
+        expect(commandFailedEvents[0].address).to.equal(commandSucceededEvents[0].address);
+      });
+    });
+  });
 });