From 0f3ae137212c59c381e59808f01f82d59502b89b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Mon, 24 Aug 2026 16:35:57 +0100 Subject: [PATCH 01/36] Toy server: support the maintenance-notification opt-in, and sending events Deliberately on RedisServer rather than a bespoke test subclass. The opt-in is an ordinary command, so any test should be able to use it, and a server that never sends a notification is the normal case - every OSS, Valkey and Garnet build behaves that way, and so will our own docker topology. The interesting behaviour is all client-side, so the fake should not be a special place. CLIENT MAINT_NOTIFICATIONS [parameter value ...] per the contract: a bare ON is valid and means "server defaults", moving-endpoint-type is the only parameter defined so far, and its five values are validated. Unknown parameters are refused rather than ignored - the client is asking the server to do something specific, and silently not doing it is worse than saying no. State is per connection, including a count, since re-arming after a reconnect is a requirement and a count is what distinguishes that from having opted in once. MaintenanceNotifications selects how the server answers: accept, reject as an unknown subcommand (what a server that never heard of it does), or reject as disabled (what one with the feature flag off does). A client has to survive all three, so a test has to be able to ask for all three. Sending covers MOVING with or without an address, the shard-scoped and slot-scoped families, explicit or generated sequence ids - the contract never defines those, so repeating one deliberately is part of what the fake owes us - and a raw-push hook for malformed frames. Notifications go only to connections that opted in; a raw push is not gated, which is the contrast the tests assert. Two bugs the tests caught: the count returned was clients *visited* rather than sent to, because ForAllClients' Action overload returns one per client regardless; and an assertion comparing against ClientCount was racy, since under RESP2 the subscription connection can register between the send and the read. --- .../MaintenanceOptInServerTests.cs | 159 +++++++++++++++++ .../StackExchange.Redis.Server/RedisClient.cs | 16 ++ .../RedisServer.Maintenance.cs | 160 ++++++++++++++++++ .../StackExchange.Redis.Server/RedisServer.cs | 77 +++++++++ 4 files changed, 412 insertions(+) create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs create mode 100644 toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs new file mode 100644 index 000000000..dcec5dcac --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs @@ -0,0 +1,159 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// The server half of the maintenance-notification contract, exercised directly. The client does not opt in +/// yet, so these drive the command as a caller would - which is also how any other test will be able to opt +/// in once the client does, since this is ordinary server functionality rather than a special test server. +/// +public class MaintenanceOptInServerTests(ITestOutputHelper log) +{ + private static InProcessTestServer CreateServer(ITestOutputHelper log) => new(log); + + private static async Task OptInAsync(IConnectionMultiplexer conn, InProcessTestServer server, params object[] args) + => await conn.GetServer(server.DefaultEndPoint).ExecuteAsync("client", args.Prepend("maint_notifications").ToArray()); + + [Fact] + public async Task BareOnIsAcceptedAndRecorded() + { + // "CLIENT MAINT_NOTIFICATIONS ON" with no parameters is explicitly valid, and means "server defaults" + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + Assert.Equal("OK", (string?)await OptInAsync(conn, server, "on")); + + var client = Assert.Single(OptedIn(server)); + Assert.Null(client.MovingEndpointType); // server defaults, not a value we invented + Assert.Equal(1, client.MaintenanceNotificationOptInCount); + } + + [Theory] + [InlineData("internal-ip")] + [InlineData("internal-fqdn")] + [InlineData("external-ip")] + [InlineData("external-fqdn")] + [InlineData("none")] + public async Task EveryDefinedEndpointTypeIsAccepted(string endpointType) + { + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + Assert.Equal("OK", (string?)await OptInAsync(conn, server, "on", "moving-endpoint-type", endpointType)); + Assert.Equal(endpointType, Assert.Single(OptedIn(server)).MovingEndpointType); + } + + [Fact] + public async Task OffClearsTheOptIn() + { + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + await OptInAsync(conn, server, "on", "moving-endpoint-type", "external-fqdn"); + Assert.Single(OptedIn(server)); + + Assert.Equal("OK", (string?)await OptInAsync(conn, server, "off")); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData("sideways")] // not on/off + [InlineData("on", "moving-endpoint-type", "sideways")] // undefined endpoint type + [InlineData("on", "not-a-parameter", "value")] // unknown parameter + [InlineData("on", "moving-endpoint-type")] // parameter with no value + public async Task MalformedOptInIsRejected(params string[] args) + { + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + var ex = await Assert.ThrowsAsync( + async () => await OptInAsync(conn, server, args.Cast().ToArray())); + log.WriteLine(ex.Message); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.UnknownSubcommand)] + [InlineData(MaintenanceNotificationSupport.Disabled)] + public async Task UnsupportingServerRejectsTheOptIn(MaintenanceNotificationSupport support) + { + // the two ways a real server refuses: it has never heard of the subcommand (OSS, Valkey, Garnet), or + // it knows it and has the feature flag off. A client has to survive both + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + await using var conn = await server.ConnectAsync(defaultOnly: true); + + var ex = await Assert.ThrowsAsync( + async () => await OptInAsync(conn, server, "on")); + log.WriteLine($"{support}: {ex.Message}"); + Assert.Empty(OptedIn(server)); + } + + [Fact] + public async Task NotificationsGoOnlyToConnectionsThatOptedIn() + { + // a real server sends to the connections that asked; sending to everything would let a client pass a + // test it should fail, by receiving notifications it never subscribed to + using var server = CreateServer(log); + await using var optedIn = await server.ConnectAsync(defaultOnly: true); + await using var notOptedIn = await server.ConnectAsync(defaultOnly: true); + + await OptInAsync(optedIn, server, "on", "moving-endpoint-type", "external-fqdn"); + var subscribed = OptedIn(server).Count(); + log.WriteLine($"{subscribed} of {server.ClientCount} connections opted in"); + + var sent = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + Assert.Equal(subscribed, sent); + Assert.NotEqual(server.ClientCount, sent); + } + + [Fact] + public async Task SequenceIdsAdvanceAndCanBeRepeated() + { + // the contract never defines these, so a client's use of them is its own invention - which means being + // able to repeat one deliberately is part of what the fake owes us + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + await OptInAsync(conn, server, "on"); + + var first = server.NextMaintenanceSequenceId; + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, 5)); + Assert.Equal(first + 1, server.NextMaintenanceSequenceId); + + // and an explicit id does not advance the counter, so a replay stays a replay + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, 5, sequenceId: first)); + Assert.Equal(first + 1, server.NextMaintenanceSequenceId); + } + + [Fact] + public async Task RawPushIsNotGatedByOptIn() + { + // the malformed-payload hook, and the contrast is the point: with nobody opted in, a notification + // reaches no one while a raw push still lands. Asserted as gated-versus-not rather than against + // ClientCount, which is read at a different instant - under RESP2 the subscription connection can + // register in between, so comparing totals is a race rather than a property + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + var gated = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + var ungated = server.SendRawPush(null, "NOT_A_REAL_KIND", "nonsense"); + log.WriteLine($"notification reached {gated}, raw push reached {ungated}"); + + Assert.Equal(0, gated); + Assert.True(ungated > 0, "a raw push should reach connections that never opted in"); + } + + private static System.Collections.Generic.IEnumerable OptedIn(InProcessTestServer server) + { + var found = new System.Collections.Generic.List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } +} diff --git a/toys/StackExchange.Redis.Server/RedisClient.cs b/toys/StackExchange.Redis.Server/RedisClient.cs index d3e2c1475..69fa5437a 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.cs @@ -99,6 +99,22 @@ public bool TryReadRequest(ReadOnlySequence data, out long consumed) } public RedisServer.Node Node => node; + + /// + /// Whether this connection has opted in to maintenance notifications, and with which endpoint type - + /// per connection, because that is how the real opt-in is scoped, so a test can assert that every + /// connection opted in rather than merely that one did. + /// + public bool MaintenanceNotifications { get; internal set; } + + /// The moving-endpoint-type this connection asked for, or null for server defaults. + public string MovingEndpointType { get; internal set; } + + /// + /// How many times this connection has sent the opt-in. Re-arming on reconnect is a requirement, and a + /// count is what distinguishes "opted in once" from "opted in again after reconnecting". + /// + public int MaintenanceNotificationOptInCount { get; internal set; } public int SkipReplies { get; set; } public void SkipAllReplies() => SkipReplies = -1; internal bool ShouldSkipResponse() diff --git a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs new file mode 100644 index 000000000..14eda7143 --- /dev/null +++ b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs @@ -0,0 +1,160 @@ +using System; +using System.Net; +using RESPite; +using RESPite.Messages; + +namespace StackExchange.Redis.Server +{ + /// + /// Sending maintenance notifications. Deliberately on the server itself rather than on a bespoke test + /// subclass: the opt-in is a normal command any test may use, and injecting a notification is a normal + /// thing any test may want to do. A server that never sends one is the ordinary case - that is what every + /// OSS build does - so the interesting behaviour is on the client side either way. + /// + public partial class RedisServer + { + /// + /// The notification types defined by the maintenance-notification contract. SMOVING and + /// SFAILING_OVER were proposed upstream but never landed, and are deliberately absent. + /// + public enum MaintenanceNotificationKind + { + /// This endpoint is being replaced; the payload names its successor. + Moving, + + /// A shard is migrating away from this node. + Migrating, + + /// The migration has completed. + Migrated, + + /// This node is failing over. + FailingOver, + + /// The failover has completed. + FailedOver, + + /// Slots are migrating (OSS cluster family). + SlotMigrating, + + /// Slots have migrated (OSS cluster family). + SlotMigrated, + } + + private static string GetName(MaintenanceNotificationKind kind) => kind switch + { + MaintenanceNotificationKind.Moving => "MOVING", + MaintenanceNotificationKind.Migrating => "MIGRATING", + MaintenanceNotificationKind.Migrated => "MIGRATED", + MaintenanceNotificationKind.FailingOver => "FAILING_OVER", + MaintenanceNotificationKind.FailedOver => "FAILED_OVER", + MaintenanceNotificationKind.SlotMigrating => "SMIGRATING", + MaintenanceNotificationKind.SlotMigrated => "SMIGRATED", + _ => throw new ArgumentOutOfRangeException(nameof(kind)), + }; + + private int _maintenanceSequence; + + /// + /// The sequence id given to the next notification, unless one is supplied explicitly. The contract does + /// not define these, so a client's use of them is its own invention - which is worth being able to + /// exercise, including by repeating one. + /// + public int NextMaintenanceSequenceId => _maintenanceSequence + 1; + + /// + /// Sends MOVING to one client, or to every client that opted in when + /// is null. A null is the documented no-address form, which a client + /// must handle whether or not it asked for none. + /// + /// The number of clients the notification was sent to. + public int SendMoving(RedisClient client, int timeSeconds, EndPoint newEndpoint, int? sequenceId = null) + => Send(client, MaintenanceNotificationKind.Moving, timeSeconds, sequenceId, newEndpoint, null); + + /// + /// Sends one of the shard-scoped notifications. is a remaining-time + /// delta and may legitimately be zero or negative for a connection that arrived mid-window. + /// + /// The number of clients the notification was sent to. + public int SendShardNotification(RedisClient client, MaintenanceNotificationKind kind, int timeSeconds, string shardIds = null, int? sequenceId = null) + => Send(client, kind, timeSeconds, sequenceId, null, shardIds); + + /// + /// Sends a slot-scoped notification (SMIGRATING / SMIGRATED) carrying a slot list in the + /// contract's comma-and-range form, e.g. "123,456,789-1000". + /// + /// The number of clients the notification was sent to. + public int SendSlotNotification(RedisClient client, MaintenanceNotificationKind kind, string slots, int? sequenceId = null) + => Send(client, kind, null, sequenceId, null, slots); + + /// + /// Sends an arbitrary push frame to a client, for the cases a well-formed notification cannot express: + /// an unknown type, a malformed payload, extra trailing elements. + /// + /// The number of clients the frame was sent to. + public int SendRawPush(RedisClient client, params string[] parts) + { + var frame = TypedRedisValue.Rent(parts.Length, out var span, RespPrefix.Push); + for (int i = 0; i < parts.Length; i++) + { + span[i] = TypedRedisValue.BulkString(parts[i]); + } + return Dispatch(client, frame, requireOptIn: false); + } + + private int Send( + RedisClient client, + MaintenanceNotificationKind kind, + int? timeSeconds, + int? sequenceId, + EndPoint newEndpoint, + string extra) + { + // [type, seqID, ...] - the sequence id is an integer, which is precisely why these frames could + // not be treated as pub/sub: element 1 is not a channel name + int count = 2 + (timeSeconds.HasValue ? 1 : 0) + (newEndpoint is not null || kind == MaintenanceNotificationKind.Moving ? 1 : 0) + (extra is not null ? 1 : 0); + var frame = TypedRedisValue.Rent(count, out var span, RespPrefix.Push); + + int index = 0; + span[index++] = TypedRedisValue.SimpleString(GetName(kind)); + span[index++] = TypedRedisValue.Integer(sequenceId ?? System.Threading.Interlocked.Increment(ref _maintenanceSequence)); + if (timeSeconds.HasValue) span[index++] = TypedRedisValue.Integer(timeSeconds.GetValueOrDefault()); + if (kind == MaintenanceNotificationKind.Moving) + { + // null rather than absent when there is no address: the client must cope with both + span[index++] = newEndpoint is null + ? TypedRedisValue.BulkString(RedisValue.Null) + : TypedRedisValue.BulkString(Format.ToString(newEndpoint)); + } + else if (newEndpoint is not null) + { + span[index++] = TypedRedisValue.BulkString(Format.ToString(newEndpoint)); + } + if (extra is not null) span[index] = TypedRedisValue.BulkString(extra); + + return Dispatch(client, frame, requireOptIn: true); + } + + private int Dispatch(RedisClient client, in TypedRedisValue frame, bool requireOptIn) + { + if (client is not null) + { + client.AddOutbound(frame); + return 1; + } + + // a real server sends only to connections that asked, so sending to all means all *opted-in*. + // Counting the sends rather than the clients visited: the Action overload of ForAllClients returns + // one per client regardless, which would report every connection as a recipient + var copy = frame; + return ForAllClients( + requireOptIn, + (target, gated) => + { + if (gated && !target.MaintenanceNotifications) return 0; + target.AddOutbound(copy); + return 1; + }); + } + } +} diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 409cbd31c..af2bd890d 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -694,6 +694,83 @@ protected virtual TypedRedisValue ClientReply(RedisClient client, in RedisReques return TypedRedisValue.OK; } + /// + /// How this server answers CLIENT MAINT_NOTIFICATIONS. Real servers vary: Enterprise supports + /// it, OSS and Valkey and Garnet do not, and Enterprise can have the feature flag off - so a client + /// must cope with all three, and a test must be able to ask for all three. + /// + public enum MaintenanceNotificationSupport + { + /// Accept the opt-in and reply +OK. + Supported = 0, + + /// Reply with an error, as a server that has never heard of the subcommand does. + UnknownSubcommand, + + /// Reply with an error, as a server whose feature flag is off does. + Disabled, + } + + /// + /// How this server answers the maintenance-notification opt-in; + /// by default, so that any test may opt in and see it accepted. + /// + public MaintenanceNotificationSupport MaintenanceNotifications { get; set; } + + // CLIENT MAINT_NOTIFICATIONS [parameter value ...], where parameter names follow a + // $type-$setting convention and moving-endpoint-type is the only one defined so far. A bare ON is + // explicitly valid and means "use the server defaults", so the parameter list is optional and + // unrecognized parameters are an error rather than something to ignore - the client is asking the + // server to do something specific, and silently not doing it would be worse than refusing + [RedisCommand(-3, nameof(RedisCommand.CLIENT), "maint_notifications", LockFree = true)] + protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, in RedisRequest request) + { + switch (MaintenanceNotifications) + { + case MaintenanceNotificationSupport.UnknownSubcommand: + return request.UnknownSubcommandOrArgumentCount(); + case MaintenanceNotificationSupport.Disabled: + return TypedRedisValue.Error("ERR maintenance notifications are disabled on this server"); + } + + bool on; + if (request.IsString(2, "on"u8)) on = true; + else if (request.IsString(2, "off"u8)) on = false; + else return TypedRedisValue.Error("ERR syntax error"); + + string movingEndpointType = null; + for (int i = 3; i < request.Count; i += 2) + { + if (i + 1 >= request.Count) return TypedRedisValue.Error("ERR syntax error"); + + if (request.IsString(i, "moving-endpoint-type"u8)) + { + movingEndpointType = request.GetString(i + 1); + switch (movingEndpointType) + { + case "internal-ip": + case "internal-fqdn": + case "external-ip": + case "external-fqdn": + case "none": + break; + default: + return TypedRedisValue.Error($"ERR unsupported moving-endpoint-type '{movingEndpointType}'"); + } + } + else + { + return TypedRedisValue.Error($"ERR unknown parameter '{request.GetString(i)}'"); + } + } + + client.MaintenanceNotifications = on; + client.MovingEndpointType = on ? movingEndpointType : null; + client.MaintenanceNotificationOptInCount++; + Log($"[{client}] maintenance notifications {(on ? "on" : "off")}, moving-endpoint-type: {movingEndpointType ?? "(server default)"}"); + return TypedRedisValue.OK; + } + [RedisCommand(2, nameof(RedisCommand.CLIENT), "id", LockFree = true)] protected virtual TypedRedisValue ClientId(RedisClient client, in RedisRequest request) => TypedRedisValue.Integer(client.Id); From 8109d4b67e8c138d820c038b48cf2c4e93ef5e2b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 14:28:34 +0100 Subject: [PATCH 02/36] Maintenance notifications: opt in, and receive (D2) (#3193) * Opt in to maintenance notifications during handshake Adds the client half of the maintenance-notification ("smart client handoffs") opt-in: a tri-state ConfigurationOptions.MaintenanceNotifications, and the CLIENT MAINT_NOTIFICATIONS ON that carries it, pipelined next to CLIENT ID. The mode names are the prescribed cross-client ones, so a connection string ports between clients - which makes Enabled mean *required* rather than merely "on". That is easy to misread, so the warning leads the XML docs on both the enum member and the property, where it shows in the completion list rather than only on hover. Enabled fails uniformly: a server that refuses, a server that answers HELLO 3 as RESP2, and a configuration that could never ask (Protocol = Resp2, or no HELLO). The last of those diverges from the letter of the spec, which mentions only the error reply - but requiring a RESP3-only feature over RESP2 is a contradiction, and half-honouring it silently is what the mode exists to prevent. Auto is the best-effort mode and never rejects a connection. Since the handshake is pipelined we don't know the negotiated protocol at write time, so the request is speculative (as the redundant AUTH already is) and ReconcileMaintenanceNotifications settles it afterwards, with every fact in hand. The opt-in processor absorbs a refusal rather than routing it through the common error path, which would raise ErrorMessage to the consumer for something we asked for on their behalf. Defaults stay Disabled globally: most servers have never heard of the subcommand, and the server is not the only thing in the path - an unrecognized CLIENT subcommand is not guaranteed to be answered as politely by a proxy as by a server. AzureManagedRedisOptionsProvider is Auto pre-emptively, which is safe precisely because Auto tolerates a refusal; pending validation against a real AMR endpoint. Also: REDIS_TESTS_MAINT_NOTIFICATIONS lets the whole suite run with the opt-in on, mirroring REDIS_TESTS_MIN_TIMEOUT_MS. Verified as a no-op at Auto against real servers that refuse it, which is the point. The toy server now matches keywords case-insensitively (we send ON, go-redis sends on, a real server takes both), and InProcessTestServer.MaxProtocolVersion can answer HELLO 3 as RESP2. * Receive maintenance notifications, and report them The other half: seven new PushKind members for the notification families, and a parser that turns them into a PushMaintenanceEvent on the existing ConnectionMultiplexer.ServerMaintenanceEvent. Observation only - nothing reacts to these yet, deliberately, so a consumer can watch what its servers announce before any behaviour depends on it. These are dispatched in OnOutOfBand *before* anything reads element 1 as a channel name, because element 1 is a sequence number rather than a channel: that is the whole reason they could not be handled as pub/sub. A frame we cannot read is consumed and forgotten rather than falling through to the command matcher, where it would take a reply belonging to something else - tested by following a malformed push with command round-trips that prove the connection is still in sync, not merely alive. Two decisions worth naming: - The type decides whether a time element is expected, not the content. A single-slot SMIGRATING payload of "123" is indistinguishable from a duration by inspection, so content-sniffing would silently lose the slot list. A notification that omits its time, or adds one where the contract says there is none, is still accepted. - MOVING's placeholder forms - explicit null, "?", and a zero port - all yield NewEndPoint = null with the raw text kept, never the answering server. Same reasoning as the unroutable-redirect work: an address that cannot be dialled must not be replaced by a guess. The PushKind lookup is now case-insensitive throughout rather than only for the new members: the pub/sub kinds are lowercase on the wire and these are uppercase, and one lookup that tolerates both beats two lookups. The shard-id and slot payloads are carried through as opaque strings. Nothing a client is asked to *do* depends on which shards are involved, and parsing a field the contract does not pin down would be inventing a model. * Cloud and on-premise defaults, and a name for them Three related pieces of configuration work, all in service of "when should SCH be on?". **MaintenanceNotifications is no longer nullable.** It followed Protocol, where null means something real ("no preference, let the library decide"). Here there is no third state - Disabled *is* off - so it now follows the convention almost every other option uses: non-nullable, falling back to the provider. That also removes three `?? Disabled` coalesces. **RedisCloudOptionsProvider**, matching the Cloud domains, with Auto. It is deliberately *not* a copy of the AMR provider, though the deployments look similar: - GetDefaultSsl stays false. AMR is TLS-only so assuming TLS there is safe; Redis Cloud enables TLS per database and plenty are plaintext, where guessing would fail their connect outright. - DefaultVersion stays at the library default. 7.4 is AMR's *floor*; Cloud still offers older versions per database, and claiming a version we do not have unlocks commands the server will reject. It does share what is about being a proxied, hosted deployment: RESP3 (which the feature requires), no configuration-broadcast channel, and fail-soft connect. **Providers can be named in a configuration string**, as `defaults=amr`, `defaults=rediscloud`, `defaults=azure` or `defaults=enterprise`. The on-premise case is why: an Enterprise cluster has whatever DNS its operator gave it, so IsMatch can never recognize it, and until now selecting a provider meant writing code - impossible for an application configured by a connection string. It also covers a hosted deployment reached behind private DNS or a proxy, where the endpoint stops looking like what it is. Hence RedisEnterpriseOptionsProvider, which matches nothing and exists to be asked for. Resolution is by name against registered providers only, never by type name: a configuration string that could name an arbitrary type would be a way to have one loaded, and would defeat trimming. Round-tripping needed care. The Defaults getter memoizes an inferred provider into the same field an explicit set writes, so merely *reading* the property would otherwise make an endpoint-derived guess indistinguishable from a decision - and re-parsing the string would then pin it. So a flag records that the caller chose it, and `defaults=` is written only when it was chosen *and* the provider has a name; unnameable custom providers behave like custom tunnels and simply do not serialize. Clone copies the field rather than the property, which is what keeps that distinction intact. Note one deliberate behaviour change: ToString() on options with an explicitly-set inbuilt provider now includes `defaults=`, where before that choice was silently dropped. DefaultsProviderProtocolNotSerialized is updated to assert both halves - that provider *values* still never leak, and that the provider *choice* now round-trips. * Provider ToString reads as its name For logs: a provider should print as "amr" rather than as a namespace-qualified type name, and an unnameable one falls back to the type as before. Note serialization deliberately does not route through this - it tests Name directly, because ToString never returns null and an unnameable provider must not end up in a configuration string. Display name and round-trippable identifier are not the same thing here, even though Tunnel conflates them behind IsInbuilt. * Maintenance notifications: relax timeouts, and read the cluster delta (D4, part of D5) (#3194) * Relax command timeouts while a server announces a disruption Stage 2 of maintenance notifications, and the first part that changes behaviour: an opening notification (MOVING, MIGRATING, FAILING_OVER, SMIGRATING) raises command timeouts for that server, and a closing one (MIGRATED, FAILED_OVER, SMIGRATED) stands them back down. Three settings, of which only the first is prescribed cross-client - the other two are ours, and say so in their XML docs: - MaintenanceRelaxedTimeout (10s): what timeouts are relaxed *to*, as a floor. The effective timeout is max(configured, this), so a caller with a generous timeout keeps it. - MaintenanceRelaxedWindowMax (3x): a backstop for a closing notification that never arrives. A window that never closes is worse than one that closes early. - MaintenancePostEventRelaxedDuration (2x, matching go-redis): a closing notification means the server-side operation finished, not that the server is back to normal latency - and completion is exactly when every other client that received the same notification re-engages. It does not apply after a cap expiry, where nothing told us the event finished and extending past the backstop would defeat it. The cap and tail derive from the *effective* relaxed timeout rather than the provider's, so raising the relaxed timeout cannot leave a cap below it; a provider can still pin either absolutely. Caught by its own test, which is why the test asserts the relationship and not just the numbers. Relaxation is server-scoped state read at sweep time, not stamped per message: both timeout sweeps rely on head-of-line ordering and stop at the first message that has not timed out, and per-message timeouts would make that short-circuit invalid - turning every heartbeat into a full scan of everything outstanding. The consequence is that relaxation covers whatever is already in flight, and stops covering it when the window closes, which is a further reason the tail earns its place. Wired into all three places a command timeout is enforced. Note the backlog sweep previously measured message age against _singleWriter.TimeoutMilliseconds - the write-lock acquisition timeout - which is about contention between writers rather than server latency; it now has its own expression, so relaxation cannot leak into lock acquisition. The sync path grows a re-wait loop, because a Monitor.Wait commits to a duration when it parks: without it, a sync caller in flight when a notification arrives would time out at the strict timeout while its async neighbour was relaxed. Also seqID dedup, per notification type, which lands here because a replayed opening notification extending a window is the first place a replay does damage. The sequence numbers are not defined by any specification, so this is deliberately conservative: an id we have already acted on is ignored, and nothing else is inferred. What is *not* relaxed, as a rule rather than a judgement call: keep-alive, the heartbeat, and connection-failure detection. Otherwise a server that died mid-maintenance would linger for the whole window, turning a latency mitigation into an availability regression. There is a test that kills a connection inside an absurdly generous window and requires the failure to still be noticed. * Report maintenance context on the faults it causes Closes out stage 2 with the fault surface: MaintenanceType on RedisTimeoutException, RedisConnectionException and FaultContext, defaulting to None. "Timeout" and "timeout during an announced failover" call for very different reactions from whoever reads the log, and until now the two were indistinguishable. Follows the established pattern on those types - Commandstatus and Flags on the timeout, FailureType on the connection fault - rather than introducing an exception type nobody catches yet, and named for the role rather than the type, as FailureType is. Both live in a partial alongside the rest of the feature, so Exceptions.cs is untouched. On FaultContext it is deliberately reported and not acted on: a fault during announced maintenance is expected and transient, and counting it towards a circuit-breaker trip that then withdraws a whole server is the opposite of what the notification was for - but "ignore faults during maintenance" is a judgement about a deployment, not about the protocol, so it belongs in policy. Also routes the effective timeout through the dead-socket heuristic in PhysicalBridge, which is derived from the command timeout and would otherwise contradict relaxation: with a relaxed timeout of 60s and that check firing at 4x the strict 5s, we would tear down precisely the connection relaxation was protecting. This is a refinement of the boundary rule, not an exception to it - socket-level failure detection is untouched, which is what actually guarantees a dead server is still noticed, and there is a test that kills a connection inside an absurdly generous window to prove it. * Read the nested cluster slot-migration payload Cross-checking our reading against the shipped clients (go-redis, redis-py) said SMIGRATED is nested, not flat: ["SMIGRATED", , [[source, target, slots], ...]] with slots a flat comma-and-range string inside each triplet. Two independent implementations agree on the nesting, which is much better evidence than our own reading of the prose - and it means we were *dropping* SMIGRATED, because the parser rejected any non-scalar element after the type. Safe, but wrong. The scalar-only guard stays for the DMC family, where nesting would signal a frame we do not understand; it is relaxed only for the two cluster kinds. A malformed triplet is skipped rather than losing the whole notification - the other triplets are still actionable, and it is what go-redis does. Exposed as ClusterSlotMigration on the event (source, target, parsed slot ranges, and the raw slot text so a list we could not parse still tells you something). Nothing acts on them yet. Two other things the cross-check turned up: - We required a readable sequence id and dropped the frame without one. go-redis length-checks these frames at two elements and reads no sequence number at all for the shard notifications, so that was stricter than a client which demonstrably works against real servers. Now a missing id costs only dedup - which is our own invention - rather than the notification. - SlotRange.TryParseInt16 was `checked`, so an out-of-range slot number threw OverflowException from a Try* method. Pre-existing and reachable today from the public SlotRange.TryParse and from CLUSTER NODES parsing; now reachable from the read loop too, where throwing is far worse than rejecting. It rejects. Fixing the parser needed two other things worth recording. RespReader's AggregateChildren() does not advance the parent reader, so MovePast(out reader) is required or the loop walks back into the children it just read and mistakes them for top-level elements. And in the toy server, Recycle() recurses, so a Standalone child inside a pooled parent is handed to a pool it never came from - Rent at every level. That one was invisible because the fake's write loop swallowed its exception into the pipe, with a reconnect covering the tracks; it now logs, which is the only reason the second bug took minutes rather than longer. * Document the maintenance options and named defaults providers Configuration.md gains the five new keys in the table, plus two sections: what a defaults provider is and why 'enterprise' exists (it cannot be detected - a self-managed cluster has whatever DNS its operator was given), and what the maintenance options do. Two things stated explicitly because they are the surprising parts: 'enabled' means *required* and will refuse a connection that cannot deliver notifications, and the maintenance durations are in seconds where every other timeout in that file is milliseconds. * Refresh topology when slots migrate away from us Reacting to SMIGRATED rather than waiting to be told by a -MOVED. This reuses the path AzureMaintenanceEvent has used for years - raise the event, then refresh - and is the whole of go-redis's SMIGRATED handling, so the risk profile is good: the worst case is the topology pass we already do. Three deliberate differences from that Azure precedent: - Scoped to triplets whose *source* resolves to us. Every node in the cluster reports the same movements, so most notifications describe somebody else, and refreshing on those means every client in the fleet re-reads topology whenever any shard moves anywhere. Resolution goes through the identity map, since a node answers to both its address and its announced hostname and the delta may name either. - Jittered by up to a second, because the fleet was all told the same thing at the same instant. Not configurable: the relaxed-window durations are options because their right value is deployment-specific and we invented them, whereas this is a fixed smear nobody needs to tune. - Cluster family only. MIGRATED and FAILED_OVER arrive in proxied deployments addressed as a single endpoint, where a refresh has no topology to learn, so they keep relaxation and nothing else. The jitter turned out to *defeat* the coalescing I was relying on: ReconfigureIfNeeded declines only while a refresh is in flight, and spreading a burst out means each pass completes before the next begins - so ten notifications became ten topology passes. Caught by the test that counts inbound CLUSTER commands. Coalescing therefore happens before the delay, via a pending flag, released when the refresh starts rather than when it finishes: anything arriving after that describes a state this pass may not have seen. Endpoints left serving no slots need no new code - the absence-based pruning from #3177 already retires them, and a refresh feeds exactly that path. It takes three generations rather than being immediate, which is slower than the HLD implies but is the existing tested policy, and is more than go-redis does at all. * Re-establish sharded subscriptions when their slots move Sharded subscriptions are slot-bound, so a slot leaving this node takes them with it. Mostly belt-and-braces: a server that migrates a slot also sends an unsolicited SUNSUBSCRIBE, and OnOutOfBand already resubscribes on that. This adds two things - it is pre-emptive where SMIGRATED arrives first, and it covers the case where the unsolicited unsubscribe never arrives or is lost, where the only other symptom is messages silently stopping, which nothing detects. It also knows *which* slots moved, so only the affected channels are touched rather than everything subscribed here. Ordinary pub/sub is not slot-bound and is deliberately left alone, even when the notification says every slot moved. Done before the refresh and without waiting for the jitter: a subscriber that is silently no longer subscribed is a correctness problem, where a stale slot map is an extra round trip. It resubscribes via *this* server rather than the migration target, reusing ResubscribeToServer unchanged - the outgoing node is the one we know has the new route, and sending there follows the redirect. The target is named in the notification and could be dialled directly, but it may be a node we have never seen or named in a form we cannot dial, and the redirect path is the one already proven by the SUNSUBSCRIBE case. Worth revisiting only with evidence. * Make the fake announce its own migrations, and fix what that exposed Migrate() only moved the slot in the fake's model - it emitted nothing, so the realistic sequence a client sees could not be reproduced. It now optionally announces itself (NotifyOnMigrate, off by default so existing tests that use Migrate to arrange a topology are undisturbed): SMIGRATING, an unsolicited sunsubscribe to subscribers of affected sharded channels, then SMIGRATED. Note that also gives the pre-existing unsolicited-SUNSUBSCRIBE path its first coverage from the fake; until now it could only be reached with a hand-built push frame. Turning it on immediately contradicted the resubscribe logic committed alongside it, in two stages: - Both signals fire for one migration, and both route to ResubscribeToServer, whose guard admits a subscription that is transiently attached to nothing. That measured six (re)subscribes for one channel. - Making it a delayed fallback - after the jitter, and only when the subscription is not attached elsewhere - fixes that. "Attached elsewhere" is the only reliable signal that the other path dealt with it; still attached to *us* is the pre-emptive case (notification beat the unsubscribe, subscription now stale) and attached to nothing is the stranded case, and both need acting on. An earlier guard on IsConnectedAny() got this wrong and silently disabled the pre-emptive case, which is the primary one. So it now costs a stranded subscription up to the jitter in recovery time, and costs nothing when it was not needed. Measured 4 (re)subscribes with notifications off and 5 with them on, the extra being the fallback acting on a subscription the unsubscribe path left attached to nothing - the feature working, not duplicate work. One thing this turned up and does not fix: after a real migration in the fake, a message published to the moved channel is not delivered - with notifications *disabled* as well, so it is either a pre-existing gap or a limitation of the freshly-added node. Deliberately not asserted, since attributing it here would blame this code for something it does not cause. Worth its own investigation. * Stop chasing a delivery failure that was never about migration The earlier note claimed a message published to a moved sharded channel is not delivered. Ran the control that should have come first: sharded pub/sub does not deliver in the fake *at all*, with no migration anywhere - SPUBLISH reports zero receivers. Evidence, for whoever picks this up: the client subscribes correctly, including following the redirect (SSUBSCRIBE on the old owner returns `-MOVED 15296 127.0.0.1:6380`, and SSUBSCRIBE then arrives on 6380), and SPUBLISH also arrives on 6380 - subscriber and publisher agree on the node. The node still answers `:0`. So the fake registers a sharded subscription somewhere its own publish lookup does not find it; a RedisChannel equality/options mismatch between the stored key and the lookup is the obvious first suspect. That makes it a gap in the fake rather than anything to do with maintenance notifications, and it means no test can currently assert sharded delivery against the toy server. The assertion and the per-node diagnostics are removed; the test keeps the property it can honestly own, which is that the resubscribe is bounded. * Toy server: sharded publish delivered to nobody, ever SPublish computed the node to filter by, with a comment saying so, and then did not pass it: var node = client.Node; // filter to clients on the same node ... PublishPair pair = new(channel, request.GetValue(2)); // node dropped ForAllClients(pair, static (client, pair) => ReferenceEquals(client.Node, pair.Node) ? client.Publish(...) : 0); PublishPair's node parameter is optional, so pair.Node was always null, ReferenceEquals never matched, and SPUBLISH answered :0 in every case - no migration required. Sharded pub/sub delivery has therefore never been covered by the fake at all, which also means a client-side sharded pub/sub bug could not have been caught here. Found while investigating an apparent "sharded subscription does not deliver after its slot migrates". It was not about migration, and it was not the client: the client followed the -MOVED correctly (SSUBSCRIBE arrived on the new owner) and routed SPUBLISH to the correct owner. The control with no migration at all is what settled it, and should have been the first experiment rather than the last. With that fixed, the end-to-end property can be asserted, so RealMigrationRecoversTheSubscriptionWithoutStorming now checks it: after a slot migration the sharded subscription delivers again. Published repeatedly, because pub/sub is fire and forget - a message published while the subscription is in flux is dropped, so losing messages during the tremor is expected while never delivering again is not, and one publish cannot distinguish them. * Retire nodes that leave the cluster, narrowed from "serving no slots" D5 asked for endpoints left serving no slots to be shut down. Narrowed deliberately: a node still listed in CLUSTER NODES is a live cluster member that may be given slots again, so dropping its connection is churn - and go-redis does not do it. Having *left* the cluster is the condition worth acting on, and the existing absence-based pruning already covers it; the notification-driven refresh is what makes us notice promptly. No client change was needed to demonstrate that, only the ability to express the scenario: RedisServer.RemoveNode removes a node the way CLUSTER FORGET does - gone from SLOTS and NODES, and every alias it answered to stops resolving. It refuses while the node still owns slots, as a real cluster does, since a topology with unowned slots says nothing useful about client behaviour. Two things this exposed, both about the policy rather than this feature: - Retirement can be starved. Pruning requires IsIdle(), which counts outstanding work, and we keep heart-beating the very node we are trying to retire - so a ping in flight makes it look busy. On a two-core runner that repeats often enough to prevent retirement indefinitely. The design notes recorded exactly this trap for the usage-based grace rule and dropped it for that reason; IsIdle() has the same problem. Excluding our own keep-alive traffic from the idleness test would fix it, and is a product change rather than something to paper over in a test - so the test is gated on a quiet machine and says why. - EndpointPruningUnitTests exercises the policy by feeding a SLOTS-only topology directly, which is not how a real refresh drives it. This test goes through ReconfigureAsync instead, which is why it sees the starvation at all. Also: this class is now non-parallel. Several of these wait out a jittered refresh, and the retirement one needs a quiet server, so sharing a machine with the rest of the suite measured as noise rather than signal. * Gate the retirement test, and record what the heartbeat theory did not explain The narrowed retirement is demonstrable on a quiet machine and fails about half the time on a two-core runner - because it does not happen, not because the test is impatient. Pruning requires IsIdle(), so something keeps the departed node looking busy. The obvious suspect was our own keep-alive: we heartbeat the very node we are trying to let go of, and an outstanding ping is enough to fail the idleness test. That theory was implemented (suppress keep-alive and the replication check for a ClusterTopology-provenance server absent from the topology) and it did *not* fix the flakiness, so the theory is wrong or incomplete and the change is reverted rather than shipped on a rationale the evidence contradicts. So the test is gated on a quiet machine with that stated, and the cause wants a focused look with instrumentation inside the pruning loop - not from a test, where an earlier attempt produced provenance readings that could not be trusted. * Record what makes a departed node look active: our own probes Measured from the snapshot the pruning loop walks, on a failing run: every term that should be true is - provenance=ClusterTopology, absentSince stable at 2, ownsSlot=False - and the blocker is outstanding work, growing ~170 per topology pass (176, 348, 520, ... 1339). That is not a keep-alive ping, which was the first theory and is why suppressing heartbeats did not help. It is the reconfigure's own autoconfigure probes to that node: it is gone, so nothing answers, and they accumulate in its backlog. IsIdle() counts backlog, so the node can never look idle - the more we look for it, the busier it appears. It only retires by winning a race on an early pass, which is exactly the load sensitivity observed. So the precondition is self-defeating in the case pruning exists for. Recorded in the test comment and the design notes; the fix is a product decision, with three candidates: exclude internal calls from the idleness measure, treat a disconnected bridge as idle since its outstanding work is doomed anyway, or stop probing servers awaiting retirement. * Idleness should count caller work, not ours Retirement requires IsIdle(), which counted *all* outstanding work. A node the topology has stopped listing still receives autoconfigure probes on every pass, and nothing answers them because it is gone, so they accumulate in its backlog: measured at ~170 per pass, growing without bound (176, 348, 520, ... 1339). The node therefore looked busy *because* we were looking for it, and could never be retired - the precondition defeated itself in precisely the case pruning exists for. The hidden internal-call flag already distinguishes our traffic from a caller's, so this is just a matter of asking the right question: IsIdle() now uses GetCallerOutstandingCount(), which walks the written-awaiting-response queue and the backlog and ignores anything flagged internal. That covers autoconfigure, handshake, and keep-alive traffic in one test, since all of it is flagged - which also disposes of the earlier keep-alive-specific theory properly rather than by suppressing heartbeats. Except that the *subscription* keep-alive was not flagged, unlike the interactive one which sets it via GetTracerMessage - so its ping, and its unsubscribe of a channel named after our own unique id, both looked like caller work. Now flagged, for consistency and because they plainly are ours. GetOutstandingCount() is unchanged and still counts everything: the retirement drain uses it, and waiting for our own in-flight probes to settle before tearing a connection down is the right behaviour there. Verified with the retirement test ungated: five clean two-core whole-suite runs, against roughly half failing before. Left alone deliberately: the availability health-check probes do not set the flag, so they still count as caller work. Arguably they should not, but IsInternalCall affects more than idleness, so that wants its own change rather than riding along here. * Clarify which subscription keep-alive actually fires The comment listed both branches without saying which one runs, and I had described them in review as though both were live traffic. Observed: against a 7.0 server the keep-alive is PING (answered with the two-element array pong), and the UNSUBSCRIBE fallback only fires against a server reporting older than 3.0, since PingOnSubscriber gates there and the default assumed version is 6.0. Also notes where the array-shaped pong is handled - IsArrayPong in OnResponseFrame - since that is not obvious from this end and is what stops the reply being taken for a pub/sub payload. * Drain on caller work too, and correct the record on the keep-alive Three corrections and one real fix, all from review. **The subscription keep-alive was already flagged.** KeepAlive has a common `if (msg != null) { msg.SetInternalCall(); ... }` after the switch, so both branches were already internal calls and the two per-branch calls added in 17151179 were redundant - removed. The commit message there claimed the subscription keep-alive "was not flagged", which is simply wrong; it was. **And the UNSUBSCRIBE branch is not "legacy".** Its condition is `IsAvailable(PING) && PingOnSubscriber`, so it is also reached when PING is disabled or renamed in the CommandMap, or fronted by something that does not support it. The version gate is only half the story. Observed: PING against a 7.0 server, UNSUBSCRIBE against one reporting 2.8. **The idleness predicate is now a predicate.** Every caller only asks whether the answer is zero, so HasCallerWork() short-circuits on the first caller message instead of counting a queue that a stalled server can leave thousands of entries long - and the interactive bridge is tested first, so the subscription bridge is usually never walked. **The real fix: the retirement drain had the same bug as IsIdle().** It looped on GetOutstandingCount(), so for a departed node - whose probes can never be answered - it always waited out the full 5s timeout before disposing. That is why the retirement test stayed intermittent after the idleness fix: measured `callerWork=False idle=True` with the node still present, i.e. retirement was being *initiated* and then blocked in drain. It now drains on caller work, still bounded by the timeout, and reports the total outstanding when abandoning since that is what is actually dropped. With both links fixed the test is ungated: 7 consecutive two-core whole-suite runs plus 4 more after cleanup, against roughly one failure in two before. Note the earlier "five clean runs" claim for the idleness fix alone was over-stated - it improved the odds without fixing the cause. * Watch a real deployment, and match the fake to what one sends toys/MaintenanceWatch: point it at a connection string and it prints what we made of every maintenance notification next to the raw payload it came from, so a misreading is visible rather than inferred. It forces RESP3 and the opt-in so it behaves the same wherever it is pointed; --enabled switches to the strict mode, which doubles as a probe for whether a server accepted the opt-in at all. Used against a Redis Cloud QA endpoint (Enterprise 8.6.2, OSS cluster API, two nodes) driven through real slot migrations. The whole chain worked with no code changes: the provider recognized the endpoint (defaults=rediscloud), the opt-in was answered +OK, and SMIGRATING/SMIGRATED were parsed - source, target and slot ranges. The captured frames, byte for byte: >3 $10 SMIGRATING :18 $9 8892-8991 >3 $9 SMIGRATED :19 *1[ *3[ $20 $18 $9 ] ] Two fidelity fixes follow from that. The fake sent the type as a *simple* string where a real server sends bulk - both parse, but there is no reason to differ. And the frames are now pinned as a regression test with the raw bytes in the comment, which is the first test in this feature backed by a capture rather than by a reading of prose. Also documents the deployment prerequisite the fault-injector console made obvious: Enterprise has a *cluster-level* flag deciding whether the subcommand exists, separate from this per-connection opt-in. A supporting version with the flag off refuses the opt-in - which Auto absorbs and Enabled turns into a refused connection, so it is worth checking before suspecting the client. * Understand a captured MOVING, and stop dedup ignoring sequence zero Captured from Enterprise 8.6.2 during a maintenance_mode scenario: >4 $6 MOVING :0 :15 _ Four elements - type as a bulk string, sequence number, a 15-second window, and an explicit RESP3 null for the address, meaning "no replacement given, reconnect the way you connected". The proxy then closed the socket, which is the whole point of MOVING: you are told to move, and then the connection goes away. That confirms two ledger items as written: the element order, and that the no-address case really is an explicit null rather than an empty string or an absent element. Our parser already read it that way. It also exposed a wart. Sequence numbers can legitimately be zero - this one was the first event of its chain - and dedup treated a stored zero as "never seen", so whichever notification opened a chain could never be recognised as a replay. The "have we seen one" state is now a separate bit. Pinned as a regression test alongside the SMIGRATING/SMIGRATED capture, including that MOVING opens a relaxed window, since that window is what covers the reconnect after the socket closes. * Sequence ids are evidence-backed now, not an invention Observed on Enterprise 8.6.2: monotonic per database, zero-based on a fresh one, shared across notification types (SMIGRATING 16 then its SMIGRATED 17), and identical on every node broadcasting a given event - so the number identifies the event rather than the connection that delivered it, which is exactly what dedup needs. The XML docs said the opposite - 'do not assume they are contiguous, or that they are scoped the same way across notification types' - so they are corrected, with the caveat that this is one build of one product and cross-deployment use stays heuristic. Also records why the per-type key is kept despite the counter being shared: within a type the ids are still monotonic, and a per-type key cannot mistake one node's earlier event for a replay of another node's later one. * One event per logical notification, and captures for the DMC family Every node broadcasts a given event with the same sequence number, so a three-proxy deployment delivered one migration three times. Collapse that: ConnectionMultiplexer.TryClaimMaintenanceEvent holds a fixed 8-slot ring of (type, sequence) and raises the public event for the first arrival only. The per-server work still runs for every copy - relaxation is per-ServerEndPoint and each connection has to open its own window - so EndPoint on the event now means "whichever node told us first", documented as such. Matched on equality rather than <=, so a lagging node reporting an earlier event we have not seen is still raised; expired by eviction rather than on a timer, since the copies arrive milliseconds apart. Also captured from Enterprise 8.6.2, closing ledger items 7 and 11: >4 $9 MIGRATING :0 :2 $6 ["27"] >3 $8 MIGRATED :1 $6 ["27"] >4 $12 FAILING_OVER :0 :2 $6 ["21"] >3 $11 FAILED_OVER :1 $6 ["21"] The opening notification carries a time and the closing one has no time element at all - which is what CarriesTime already assumed - and the shard list is a stringified JSON array of id strings. Pinned as tests; the fake can now omit the time, which it previously always sent. That test found a fake-server bug: Dispatch handed the same rented frame to every opted-in client, the first client's write loop recycled it, and the second faulted with "Array element cannot be nil" and lost its connection. So every broadcast had only ever reached one client, which made multi-node fan-out untestable. Built per recipient now, sequence id computed once. * Model the server's catch-up channel in the fake Redis Enterprise retains the most recent shard-scoped completion and replays it to each connection that opts in, coalesced into the same read as the +OK. The boundary is sharp (RS 8.0.22): MIGRATED and FAILED_OVER are retained; MIGRATING, FAILING_OVER, MOVING, SMIGRATING and SMIGRATED are not. What fits all seven is "the completion of a shard-scoped event" - the two carrying an affected-shards list. That gives the design property the handoff work depends on: the catch-up channel can only ever say "a disruption ended", never "one is starting". MOVING, the only notification demanding action, is never replayed - so a reconnecting client cannot be told to hand off by a stale frame, and D6 needs no staleness guard to be safe from replay. Asserted as a negative over all five non-retained kinds. Retention is most-recent-replaces, never a queue, so a connection sees at most one. RetainCompletions turns it off for tests that would rather not reason about which kinds are retained. Needed a deferred-outbound slot on RedisClient: the read loop enqueues a command's reply after Execute returns, so a handler calling AddOutbound directly puts its frame *before* its own +OK, which is the wrong order. The replay also gets us the first test of a push frame arriving mid-handshake, interleaved with our own handshake replies - previously every notification arrived on a settled connection. Not implemented: the catch-up-aware skip of the topology refresh. SMIGRATED turns out not to be retained, so no retained frame can reach the refresh path, and the guard would be unreachable code. * Probe traffic is not caller work, and drop a ValueTuple from the library Health-check probes counted as work a caller is waiting on, so an endpoint being probed looked busy - and idleness is what decides whether an endpoint that has left the deployment can be retired. Invisible while probes only ran under MultiGroupMultiplexer; a blocker for enabling them anywhere retirement also runs. Deliberately not the internal-call bit, which would have been one line: that flag also decides queuing, bypassing the backlog and queuing while disconnected regardless of policy. A health check that bypasses the backlog cannot see a bridge whose queue is not draining, which is the signal geo-redundant failover depends on. So this is a separate bit (19), and the four accounting sites now ask IsCallerFacing rather than !IsInternalCall. The flag has to be on UserSelectableFlags, because probes reach the pipeline through the public API and the constructor masks anything else - so it arrives as caller-supplied flags or not at all. A caller passing it only opts their own command out of idleness accounting. Exposed as HealthCheckContext.ProbeFlags so a third-party probe can be correct too. An injecting IDatabase wrapper would make that automatic, and [AutoDatabase] would make it mechanical, but it would also make the flag invisible and un-opt-out-able, and a probe that forgets it merely reproduces today's behaviour. Also: the dedup ring added yesterday used a tuple, which pulled ValueTuple into the library and broke SanityChecks.ValueTupleNotReferenced - .NET Framework consumers would need the package. Named struct instead. That is already pushed on this branch, and my filtered test runs hid it; the full suite is what caught it. * Maintenance notifications: the MOVING resolve primitive, and a fake that closes the socket (part of D6) (#3202) * The fake's MOVING closes the socket, on the measured timing MOVING's defining half was missing: the socket goes away. Measured on RS (2026-08-28) the close lands at +18.4s and +16.6s against a declared 15s window, so the window is a floor with slack rather than a deadline - the fake defaults to announced-plus-slack, and tests assert that we act within the window, never that the socket survives to the end of it. A shorter delay is available to exercise a less generous proxy. Blast radius is the node, not the connection: four connections to one node differing only in handshake all closed simultaneously, and only the opted-in ones were warned. So the close is scoped to siblings sharing a node, and D6 will reuse RetireAsync rather than anything narrower. A zero delay is deliberately not a case: it races delivery of the notification itself, which no real timing produces. My first version of this test asserted it, and it failed for that reason. * The MOVING re-resolve loop: poll DNS past the address being retired Measured behaviour makes this a poll, not a lookup. Relative to the notification: the endpoint moves server-side at +8.6s, DNS follows at +9.7s and +4.4s across two runs, and the sockets close at +18.4s and +16.6s - against a declared 15s grace and a 5s TTL. So the first answer names the address we were just told to leave, in every run observed, and a client that treats it as authoritative hands off to the node it is trying to escape. The short TTL is what makes polling work: several attempts fit in the window. MovingEndpointProbe is deliberately pure - the caller supplies the resolver, the interval and the budget - because the alternative is untestable: no in-process fake can move a DNS record. Jitter stays at the call site with the existing refresh jitter. Returning null when the window expires is a result, not a failure: the server closes the socket anyway and the relaxed window covers the reconnect, so guessing an address would be worse than doing nothing. Seven tests, including the ones that matter: DNS trailing the notification, a resolution blip mid-handoff, a round-robin record naming both nodes at once, and a zero window still getting one attempt ("act now", not "do nothing"). * Multi-address hostnames are the common case, so stepping sideways is the norm Measured 2026-08-28, all on a 5s TTL: all-nodes 2 A records, all-master-shards 3, single 1 - and an all-master-shards database whose shards shared a node also resolved to 1. So the count follows actual proxy placement rather than the policy name, and `single` (the shape the MOVING timeline was measured on) is the unusual one. The rule survives unchanged, which is the useful part: "take any address that is not the one being retired". With several records the first resolution already names a live sibling proxy, so the handoff steps sideways at once rather than waiting ~9s for DNS - any proxy of the same database serves the same data. The poll only engages when the record names nothing but the retiring address, which is exactly where waiting is the only option. Nothing reads the policy, so placement-driven counts need no special case. Two tests pin the branches, and the log now distinguishes them, because "stepped sideways to a sibling that was already advertised" and "the record has moved to the replacement" look identical otherwise and mean different things when someone is debugging a handoff. Note a full-fleet operation can hand us a sibling that is also about to be retired, so handoffs can chain. Self-limiting - each MOVING carries its own window and relaxation - but worth recognising rather than mistaking for a loop. * Ask whether we are still advertised, and fix a race in my own test The measured gap this closes: on a multi-proxy database, taking a node out on the *shrink* path announces nothing about the endpoint, drops the victim from DNS at +21.4s, and closes its socket silently at +34.7s. So for thirteen seconds the condition is plainly visible to anybody who asks - our address is no longer advertised - and the client's only other signal is a socket dying with no explanation. MIGRATED lands ~5s before DNS moves and is the one notification the server retains, which makes it the prompt to ask on. IsStillAdvertisedAsync returns bool?, and the null carries weight: a resolution failure, or a record momentarily resolving to nothing, is "cannot tell" and must never become "give it up", or one DNS blip recycles every healthy connection at once. MovingEndpointProbe -> AdvertisedAddressProbe: the name stopped describing it once it answered two questions. Both reduce to "what does the record say now, and is my address in it", which is why this is one primitive and not two. Also fixes MaintenanceOptInClientTests.OptInIsReArmedOnReconnect, which asserted *client* state immediately after observing *server* state: the server counts the opt-in when it processes the request, we mark the feature live when we read the reply, a beat later. Intermittent under two cores, mine, and on a branch that was already pushed - so it would have surfaced in CI rather than here. Six consecutive clean runs after polling for the client side. * MOVING fires when the address set gains a member, and DNS may lose the race Nine observations now fit one rule: MOVING is emitted when the endpoint's address set GAINS a member, and is silent when it only loses members. Policy narrowing, maintenance_mode, a 3->2 exclude and a reduction to a single proxy all only shrink, and all were silent; a substitution on that surviving single proxy announced. So the discriminator is neither "single proxy" nor placement. The consequence for the probe is a third outcome, now documented as measured fact rather than as a defensive branch. On one cluster DNS was correct 4.4-9.7s after MOVING, comfortably inside the 15s grace; on another it updated at +18.7s, three seconds AFTER the socket closed at +15.7s. So "window expired with the record still stale" is normal, and the only move left is to reconnect after the close and resolve then - which for a hostname endpoint is already correct. Anybody reading the null return as unreachable would be deleting the handling for a case that happens. Also recorded why the rule stays "any address that isn't mine" rather than "prefer a newly appeared address", despite MOVING marking precisely the moment something joins: a live sibling is at least as good and is available now, while the newcomer is invisible until the record updates. Preferring it means waiting, and waiting is the failure mode. Replacement proxies were measured accepting connections at +6.3s while DNS still advertised only the retiring node - which is a good argument for remembering addresses, deferred because a remembered address whose port was reassigned would be a silent wrong-server connection and a proxied standalone gives us no identity check to catch it. * Maintenance notifications: act on MOVING, and test it against a real deployment (rest of D6, and D9's dedicated testing) (#3203) * A fault-injector test tier: one folder in, databases provisioned per shape New project, tests/StackExchange.Redis.FaultInjector.Tests, net10.0 only - these tests are about server behaviour, not our down-level targets. Picked up by Build.csproj's glob so it compiles in CI, but CI's test step names the main project explicitly, so it never runs there; build.ps1's traversal does run it, which is why the skip behaviour has to be right. One path is the whole configuration: SER_FI_CONFIG_DIR (or the console's FI_CONSOLE_CONFIG_DIR) points at the directory already mounted into the injector as /app/config, so cluster credentials, the CA certificate and the compose file are all found rather than hand-carried into the run. Three states, deliberately distinct: no directory skips; a directory without E2E_SCENARIO_TESTS=true skips (these create and delete real databases); and configured-and-meant-but-broken FAILS. The third is the point - a suite that skips on a broken environment reports success for tests that never ran, and gets trusted at exactly the wrong moment. All three verified. Databases are provisioned by the tests, per shape rather than per test, which removes the conveyance problem entirely: a test that asked for oss_cluster knows what it asked for, so endpoints.json stops being the source of truth for per-database facts and its missing oss_cluster/endpoint_type fields stop mattering. Shapes exist because they change behaviour - A-record count follows proxy placement, and the handoff branches on whether a live sibling exists. Named sertest--. Cleanup is per fixture and unconditional; the startup sweep matches the sertest- prefix and nothing else, so it can never touch a database created by hand. Port collisions retry upward, as go-redis has to. TLS trusts the environment's CA via TrustIssuer. If the CA is missing, TLS tests fail rather than disabling validation: a TLS test that quietly stops checking identity reports success for the one thing it exists to catch. Two traps from the console's known-gaps are encoded rather than left to be rediscovered: poll on pending AND running (a loop waiting only on pending returns while the job is still going), and setup_id lives in the injector's memory so teardown keeps a bdb_id fallback. Teardown also runs on cancellation, with its own budget - the one place the ambient test token must not apply. Unverified and flagged in the README: the create_database parameter names are the injector's prose-documented wire schema, gathered in one place so a real run can correct them. * Prove the fault-injector tier live, and narrow the MOVING rule Run against a real RS 8.0.22 deployment: both template databases connect, negotiate RESP3 and report the opt-in active, and all four topology-change-standalone scenarios run end to end in 7m32s with the notifications observed and parsed. Cleanup verified - four scenarios left the cluster with exactly its two original databases. The rule the measurements produced is a conjunction, narrower than either half: MOVING fires when the connection's own proxy LEAVES the endpoint's address set AND the set GAINS a member. The counter-example is dns_resolution_change, which widens single -> all-master-shards: addresses are plainly added, yet nothing is announced, because the client's proxy is not going anywhere - and then the proxy restarts and the socket closes at +44.5s with no warning. That also resolves what looked like a contradiction, maintenance_mode announcing on a single-proxy database but not on a multi-proxy one: with one proxy, moving it *is* a substitution. The scenario expectations encode this, silence included, so a build that starts announcing the widening case tells us rather than passing quietly. Three more findings. The window overshot again, by 19.1s and 17.5s against a declared 15s, so "floor with slack" has four independent measurements and no counter-example. The sequence counter is shared across all types including MOVING (0, 1, 2 in one chain), which the per-type dedup already assumed. And data_movement_no_conn_drop moved shards with both notifications delivered and the connection never disturbed, so MIGRATING does not imply an impending disconnect. Corrections to the harness from real responses, replacing guesses: - scenario setup provisions its own database and returns setup_id, bdb_id, db_name, endpoints, password, tls, mtls_files and config in ~12s, so scenario tests need neither create_database nor endpoints.json nor the REST API - every trigger publishes the dbconfig it requires, and all four want proxy_policy: single, which no template creates - hence setup provisioning - setup_id is a handle, not an action id: polling /action/{setup_id} 404s - the create_database schema now matches bdb_config.json, which disambiguated oss_cluster_api_preferred_endpoint_type (ip vs hostname, and therefore whether a TLS client can verify its targets) from ..._preferred_ip_type (internal vs external routing) - I had conflated them Traversal run with no environment configured: 7 skipped, everything else green. * Cover the injector's scenario families, and sort what is left into buckets Ran the fault injector's scenarios against the live RS 8.0.22 cluster and added the ones that hold their value as tests. Green live: the OSS cluster family (SMIGRATING/SMIGRATED parsed to source -> target, with 1440 reads and zero failures across a real shard migration), sharded subscriptions recovering unaided after a migration - D5's resubscription, previously fake-only - the failover pair (FAILING_OVER seq=0 time=2s ["52"], FAILED_OVER seq=1) received end to end for the first time, and proxy restart recovery. Four schema facts the injector taught us, each replacing a guess: - create_database wants its config nested under "database_config"; a flat payload is rejected with "got None" - sharding requires shard_key_regex, or Redis Enterprise refuses the database with "Invalid sharding configuration" - /slot-migrate/setup's trigger is how to *provision* (only "reshard"), not how to migrate, and its effect enum is narrower than the discovery endpoint's - remove-add cannot be set up at all - setup provisions a database and returns it, so scenario tests need neither create_database nor endpoints.json Also two harness fixes worth their own mention. The create retry loop retried everything, so "missing shard_key_regex" arrived eight times over half a minute instead of once; it now retries only port collisions. And the traceback summariser split on '\n' when the injector's JSON carries the two characters backslash-n, so every skip message was a wall of Python. Scenarios this cluster cannot produce - add and slot-shuffle need a node with several shards, and three nodes with sparse placement give one each - now skip on a matched message rather than failing. Deliberately not run while unattended: shard/node/proxy/cluster failure, node_remove and reset_cluster, which damage or reset the cluster. The four-bucket assessment is in the notes: what works, what this feature still owes (D6's action half, the connect-failure trigger, moving-endpoint-type, MAINT_NOTIFICATIONS_INFO), what already works outside the feature, and what is untested - of which network_latency matters most, because it is how timeout attribution finally gets live evidence. * Reach the TLS variant, and diagnose why it cannot run here include_tls does not request TLS - it widens the list of variants setup may choose from, and variant_index picks one. With no flags a trigger offers one variant (single), with include_tls two (single, single_tls), with include_mtls a third (mtls). Passing include_tls alone provisions variant 0 and yields a plaintext database, which is how the first attempt skipped itself. With variant_index=1 the database came up TLS-enabled and the connect was refused: the remote certificate was rejected by the validation callback. That is the environment, not us - the folder's server certificate covers *.marcgravell-test-46be1d08... while the live cluster is marcgravell-test-e21cd75d..., left over from an earlier provision and three days older than the env_output.json beside it. Our behaviour was right: TrustIssuer tolerates chain errors only, so a name mismatch fails outright, which is the whole point of it. So the test now compares the certificate's DNS names against the cluster name *before* provisioning anything, and skips naming both. Without that, a stale certificate reads as a client bug and costs somebody an hour of certificate archaeology; the check costs nothing and happens before a database exists. Also set AbortOnConnectFail=true in the TLS test only. Everywhere else tolerating a slow start is right, but with it false a certificate problem is indistinguishable from a slow cluster: ConnectAsync succeeds, IsConnected is false, and the reason is gone. That is exactly how the first failure presented. Traversal with no environment: 15 skipped, everything else green. * D6: act on MOVING instead of waiting to be disconnected Today a MOVING is survivable - the socket closes and we reconnect - but the announced window goes entirely unused, and the reconnect re-resolves to whatever DNS says at the moment the server chose, which has been measured as still naming the node being retired. This uses the window: wait for DNS to move, then pick the moment ourselves. The dispatch turns on the form of the endpoint, which also corrects the earlier assumption that MOVING should reuse endpoint retirement: - hostname, no successor (every observed MOVING): the ServerEndPoint stays, only the address behind the name moves, so retiring it would delete our only route to the deployment. Probe until the record moves, then recycle the connections so they re-resolve. - address with a named successor: genuinely a different endpoint, so re-read the topology. Never observed - eleven routes, all explicit nulls - so this exists because the contract has it. - address, no successor: nothing to re-resolve and nowhere named to go. Doing nothing is correct. Deciding is separated from acting so the decision can be tested exhaustively without a server: DecideAsync takes the endpoint, the current address, the window and a resolver. That seam exists because the whole thing turns on DNS *changing*, which no in-process fake can arrange - ConnectionMultiplexer.AddressResolver defaults to real DNS. Recycling is a dispose: that already routes through RecordConnectionFailed to OnDisconnected, which reconnects immediately, so there is no new lifecycle to get wrong. Both bridges, because the measured blast radius is the node. Drained first, bounded by what is left of the window - the socket dies at the end regardless, so anything undrained was going to fail either way and draining strictly dominates. Jitter is a fraction of the window rather than a flat delay, capped at a second. A 2s window - which the shard notifications really do announce - must not spend half of itself waiting, and a 15s window does not justify a long wait when DNS has been seen moving after four seconds. Also safe from replay by construction, which is why there is no staleness guard: the server retains only shard-scoped completions, so a MOVING is never delivered as catch-up. Nine tests: five decision branches, jitter bounds, and an end-to-end recycle against the fake with MovingClosesConnection deliberately off, so the only thing that can replace the connection is our own handoff. Three consecutive two-core Release runs: 6215 passed, 0 failed. * D6 proven live, and the feedback loop it exposed On the real cluster the handoff does what it was built for: conn_drop/endpoint_rebind MOVING +9.3s -> recycled and reconnected +9.5s server would have closed at +25.5s maintenance_mode MOVING +21.7s -> recycled and reconnected +21.9s server closed at +38.0s So we move roughly sixteen seconds before being pushed, on both routes. The first live run also found a bug that no fake could have produced: a server re-sends MOVING to a connection that opts in while the window is still open. Since the handoff replaces the connection, acting on the repeat loops - recycle, reconnect, get told again, recycle - and it produced twelve recycles from a single event. OnMaintenanceWindowOpened already claimed the sequence id and knew it was a repeat; the handoff was not asking. It now returns whether the notification was new and the handoff gates on it, and the live test asserts *exactly one* recycle. Second finding, recorded rather than fixed: our own recycle does not raise ConnectionFailed, because disposal is not reported as a failure. From outside the library a handoff is therefore invisible - an operator sees a reconnect with no reason given. HandoffRecycles and LastHandoffOutcome exist because Multiplexer.Trace is [Conditional("VERBOSE")] and compiles away, so there would otherwise be no record at all of what a handoff decided. Whether it should surface something publicly is a real question, not settled here. Two consecutive two-core Release runs: 6215 passed, 0 failed. * Report a handoff as MaintenanceHandoff rather than silently A handoff was invisible from outside the library: the replacement connection raises ConnectionRestored, but our own recycle raised nothing, so a consumer tracking connection state saw a restore with no matching failure and no reason for the churn. The reporting block is gated on "if (_ioStream is not null || isInitialConnect)" - if *we* didn't burn the pipe, flag it - and Dispose runs Shutdown first, which is precisely why an ordinary dispose is silent. So the fix is ordering: record the failure while the pipe is still live, then dispose. ConnectionFailureType.MaintenanceHandoff is the right home. The existing event args already carry endpoint, connection type and a discriminator, and CircuitBreaker is the precedent for a deliberate client action reported this way. Documented for what it is: consumers alerting on ConnectionFailed should filter it out, since it means planned maintenance rather than a fault - and the test asserts we report *only* that, never SocketFailure or SocketClosed, so planned maintenance cannot end up in fault dashboards. Four consecutive two-core Release runs at 6215 passed. Note one earlier run reported two failures whose names I did not capture and which have not recurred in four runs since; if they come back I will capture them properly rather than guess. * Fix the cluster-flag call, which had been failing silently update_cluster_config wants its flags nested under "config" - the same shape create_database wants for "database_config" - and a flat payload is rejected with "Invalid parameter 'config': got None". Because the call is best-effort and only wrote to Console, it failed silently for a full day of testing without anybody noticing. Note the impact was small: the environment templates enable these flags at provision time, so this call is a safety net rather than the mechanism, and every test was passing on its own merits. It matters for an environment provisioned without them, where the alternative is every test failing at connect and blaming the client for a server-side setting. Verified corrected against the live injector. Fixture diagnostics now go to a collected SetupLog as well as the console, since a fixture has no test output helper and console writes are exactly what got lost. * Reach the migrations and the TLS variant that were being skipped Three "environment limitations" turn out to have been mine. add and slot-shuffle were skipping with "No node with multiple shards found", and remove-add was unreachable because /slot-migrate/setup's effect enum excludes it. The cluster was never the problem: the setup leg provisions one shard per node, so there is nothing to move a shard *from*. Provisioning our own database - six shards, dense placement, two per node over three nodes - and driving the run leg by bdb_id makes all three run, and all three now pass live. The generalisation is the useful part: a scenario setup cannot arrange is still reachable by provisioning the database ourselves. remove-add is the best of them: it moves every shard as five SMIGRATING/ SMIGRATED pairs sharing one sequence chain (0-9), which exercises the dedup and the event collapse far harder than a single migration. The first dense attempt failed for an unrelated reason: the client tried to reach 10.0.101.15, a VPC-private address, because a cluster database created without oss_cluster_api_preferred_ip_type defaults to *internal* - so CLUSTER SLOTS advertised addresses unreachable from outside the VPC. That is the field separated earlier from endpoint_type, biting for real: ip_type is internal-versus-external routing, endpoint_type is ip-versus-hostname identity. Shapes now default to external. TLS also runs now that the environment's certificates match the cluster: connected with validation on, opt-in active, MOVING received over TLS. My own mismatch detector had a bug worth recording - it read one name via GetNameInfo and compared it against the cluster name, so a wildcard "*." never matched, and it skipped a perfectly good environment. These certificates carry both a wildcard and the bare name, the hosts dialled are database endpoints *under* the cluster domain, and a wildcard correctly does not match its own parent. It now reads the full SAN list. Cluster left with exactly its two original databases. Local suite: 6215 passed. --- Directory.Build.props | 2 +- docs/Configuration.md | 56 ++ docs/exp/SER010.md | 58 ++ src/RESPite/Shared/Experiments.cs | 1 + .../Availability/FaultContext.cs | 16 +- .../Availability/HealthCheckContext.cs | 14 +- .../Availability/HealthCheckProbe.Ping.cs | 4 +- .../HealthCheckProbe.StringSet.cs | 6 +- .../ClusterConfiguration.cs | 90 +- .../AzureManagedRedisOptionsProvider.cs | 18 + .../Configuration/AzureOptionsProvider.cs | 3 + .../Configuration/DefaultOptionsProvider.cs | 112 +++ .../RedisCloudOptionsProvider.cs | 83 ++ .../RedisEnterpriseOptionsProvider.cs | 50 ++ .../ConfigurationOptions.cs | 178 +++- .../ConnectionMultiplexer.Events.cs | 63 ++ .../ConnectionMultiplexer.cs | 20 +- .../Enums/ConnectionFailureType.cs | 17 + src/StackExchange.Redis/ExceptionFactory.cs | 5 + src/StackExchange.Redis/LoggerExtensions.cs | 8 +- .../Maintenance/AdvertisedAddressProbe.cs | 227 +++++ .../Maintenance/ClusterSlotMigration.cs | 53 ++ .../Maintenance/MaintenanceFaultSurface.cs | 47 + .../Maintenance/MaintenanceHandoff.cs | 133 +++ .../MaintenanceNotificationType.cs | 58 ++ .../Maintenance/PushMaintenanceEvent.cs | 126 +++ .../MaintenanceNotificationMode.cs | 50 ++ src/StackExchange.Redis/Message.cs | 41 +- src/StackExchange.Redis/PhysicalBridge.cs | 66 +- .../PhysicalConnection.Maintenance.cs | 304 +++++++ .../PhysicalConnection.Read.cs | 33 +- src/StackExchange.Redis/PhysicalConnection.cs | 32 +- .../PublicAPI/PublicAPI.Unshipped.txt | 65 ++ src/StackExchange.Redis/RedisLiterals.cs | 3 + src/StackExchange.Redis/ResultProcessor.cs | 39 + .../ServerEndPoint.Maintenance.cs | 591 +++++++++++++ src/StackExchange.Redis/ServerEndPoint.cs | 54 +- .../StackExchange.Redis.csproj | 2 + .../ClusterFamilyScenarioTests.cs | 218 +++++ .../DenseClusterScenarioTests.cs | 109 +++ .../Environment/CertificateSanity.cs | 94 ++ .../Environment/ClusterRestClient.cs | 71 ++ .../Environment/DatabaseShape.cs | 129 +++ .../Environment/ExistingDatabase.cs | 151 ++++ .../Environment/ExistingDatabaseFixture.cs | 138 +++ .../Environment/FaultInjectorEnvironment.cs | 179 ++++ .../Environment/FaultInjectorFixture.cs | 186 ++++ .../Environment/ProvisionedDatabase.cs | 170 ++++ .../FaultInjector/FaultInjectorClient.cs | 204 +++++ .../FaultInjector/ScenarioRun.cs | 257 ++++++ .../FaultInjector/ScenarioSupport.cs | 95 ++ .../MovingHandoffScenarioTests.cs | 131 +++ .../Poll.cs | 21 + .../ProxyAndFailoverScenarioTests.cs | 169 ++++ .../README.md | 60 ++ .../RealDeploymentSmokeTests.cs | 64 ++ ...kExchange.Redis.FaultInjector.Tests.csproj | 25 + .../TlsScenarioTests.cs | 161 ++++ .../TopologyChangeScenarioTests.cs | 164 ++++ .../AdvertisedAddressProbeTests.cs | 216 +++++ .../ClusterSlotMigrationUnitTests.cs | 61 ++ .../StackExchange.Redis.Tests/ConfigTests.cs | 74 +- .../DefaultOptionsTests.cs | 129 +++ .../HealthCheckPolicyUnitTests.cs | 30 +- .../Helpers/TestConfig.cs | 26 + .../InProcessTestServer.cs | 9 + .../MaintenanceHandoffTests.cs | 113 +++ .../MaintenanceNotificationTests.cs | 833 ++++++++++++++++++ .../MaintenanceOptInClientTests.cs | 251 ++++++ .../MaintenanceRelaxationTests.cs | 322 +++++++ .../MaintenanceTopologyRefreshTests.cs | 400 +++++++++ .../RedisValueStorageKindUnitTests.cs | 4 +- tests/StackExchange.Redis.Tests/TestBase.cs | 2 +- toys/MaintenanceWatch/MaintenanceWatch.csproj | 15 + toys/MaintenanceWatch/Program.cs | 115 +++ .../RedisClient.Output.cs | 27 + .../StackExchange.Redis.Server/RedisClient.cs | 7 +- .../RedisServer.Maintenance.cs | 283 +++++- .../RedisServer.PubSub.cs | 36 +- .../StackExchange.Redis.Server/RedisServer.cs | 63 +- 80 files changed, 8479 insertions(+), 61 deletions(-) create mode 100644 docs/exp/SER010.md create mode 100644 src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs create mode 100644 src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs create mode 100644 src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs create mode 100644 src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs create mode 100644 src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs create mode 100644 src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs create mode 100644 src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs create mode 100644 src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs create mode 100644 src/StackExchange.Redis/MaintenanceNotificationMode.cs create mode 100644 src/StackExchange.Redis/PhysicalConnection.Maintenance.cs create mode 100644 src/StackExchange.Redis/ServerEndPoint.Maintenance.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/README.md create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs create mode 100644 tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs create mode 100644 tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs create mode 100644 toys/MaintenanceWatch/MaintenanceWatch.csproj create mode 100644 toys/MaintenanceWatch/Program.cs diff --git a/Directory.Build.props b/Directory.Build.props index f5981dc8b..c0863db83 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,7 +11,7 @@ $(MSBuildThisFileDirectory)Shared.ruleset NETSDK1069 - $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009 + $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009;SER010 https://github.com/StackExchange/StackExchange.Redis/releases https://seredis.dev/ MIT diff --git a/docs/Configuration.md b/docs/Configuration.md index 66eaa20ab..0c9afc388 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -100,6 +100,11 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | setlib={bool} | `SetClientLibrary` | `true` | Whether to attempt to use `CLIENT SETINFO` to set the library name/version on the connection | | protocol={string} | `Protocol` | `null` | Redis protocol to use; see section below | | highIntegrity={bool} | `HighIntegrity` | `false` | High integrity (incurs overhead) sequence checking on every command; see section below | +| defaults={string} | `Defaults` | `null` | Selects a named defaults provider; see section below | +| maintNotifications={string} | `MaintenanceNotifications` | `disabled` | Whether to ask servers for maintenance notifications; see section below | +| maintRelaxedTimeout={int} | `MaintenanceRelaxedTimeout` | `10` | **Seconds** that command timeouts are relaxed to during announced maintenance; see section below | +| maintRelaxedWindowMax={int} | `MaintenanceRelaxedWindowMax` | `30` | **Seconds** a relaxed window may last at most; see section below | +| maintPostEventRelaxed={int} | `MaintenancePostEventRelaxedDuration` | `20` | **Seconds** timeouts stay relaxed after maintenance completes; see section below | Additional code-only options: - LoggerFactory (`ILoggerFactory`) - Default: `null` @@ -287,6 +292,57 @@ config.ReconnectRetryPolicy = new LinearRetry(5000); //6 5000 ``` +## Defaults providers + +Some settings have sensible values that depend on *what you are connecting to* rather than on what you want, so +the library keeps them in a provider and consults it for anything you have not set explicitly. Providers are +normally chosen automatically by looking at the endpoints - an `*.redis.cache.windows.net` host selects the +Azure provider, and so on - and you can select one explicitly instead: + +``` +myserver:6379,defaults=enterprise +``` + +The names are `azure`, `amr` (Azure Managed Redis), `rediscloud` and `enterprise` (a self-managed Redis +Enterprise deployment). The last one exists because it cannot be detected: a self-managed cluster has whatever +DNS its operator gave it, so there is nothing to recognize. It is also the right choice for a hosted deployment +reached behind private DNS, a CNAME or a proxy, where the endpoint no longer looks like what it is. + +A provider chosen explicitly appears in `ToString()`; one that was merely inferred from the endpoints does not, +since writing it out would turn a guess into a decision. Custom providers (assigned in code, via +`ConfigurationOptions.Defaults`, or registered with `DefaultOptionsProvider.AddProvider`) work exactly as +before; they can additionally be named in a configuration string if they override `Name`. + +## Maintenance notifications + +Redis Enterprise and Redis Cloud can warn a connected client *before* a disruptive event - a shard migration, a +failover, or an endpoint being replaced - so the client can act ahead of it rather than discover it by way of a +broken connection. This requires RESP3, and the client asks for it per connection: + +| Mode | Behaviour | +| --- | --- | +| `disabled` | Never ask. The default, because most servers have never heard of the request. | +| `auto` | Ask, and carry on if the server refuses or the connection ends up RESP2. Safe against a mixture of servers, and what most callers want. | +| `enabled` | **Required**: ask, and reject the connection unless notifications are live. Only point this at a deployment you know supports them. | + +Note that `enabled` means *required*, not merely *on* - it is the cross-client name for that mode, and it will +refuse a connection that cannot deliver notifications, including any RESP2 connection. The `amr`, `rediscloud` +and `enterprise` providers select `auto` for you, so on those deployments you need not set anything. + +While a disruption has been announced, command timeouts are relaxed - raised to `maintRelaxedTimeout`, never +lowered, so a caller with a more generous timeout keeps it. The window ends when the server says the disruption +finished, and then stays relaxed for `maintPostEventRelaxed` longer, because completion is exactly when every +other client that received the same notification comes back. `maintRelaxedWindowMax` bounds a window whose +closing notification never arrives. Only *command* timeouts are affected: keep-alive and connection-failure +detection are untouched, so a server that dies mid-maintenance is still noticed on the usual schedule. + +These durations are in **seconds**, unlike every other timeout here, because those are the units the +cross-client specification names for them. + +Receiving a notification raises `ConnectionMultiplexer.ServerMaintenanceEvent` with a `PushMaintenanceEvent`, +and a timeout or connection fault that happened during an announced disruption carries a `MaintenanceType` +saying so. + ## Redis protocol RESP3 is a newer protocol (available on v6 servers and above) which allows (among other changes) pub/sub messages to be communicated on the *same* connection - which can be very diff --git a/docs/exp/SER010.md b/docs/exp/SER010.md new file mode 100644 index 000000000..2645b9393 --- /dev/null +++ b/docs/exp/SER010.md @@ -0,0 +1,58 @@ +Maintenance notifications ("smart client handoffs") are a server feature by which a server warns a +connected client in advance of a disruptive event - a shard migration, a failover, or the endpoint +itself being replaced - so the client can act before the disruption rather than react to a broken +connection afterwards. The client opts in per connection with `CLIENT MAINT_NOTIFICATIONS ON`, and the +notifications then arrive as RESP3 push frames on the connection that carries commands. + +The feature is experimental here for three separate reasons: + +1. **The server side is not universal.** Only Redis Enterprise and Redis Cloud emit these + notifications today; OSS Redis, Valkey and Garnet do not recognize the opt-in at all. That is why + `MaintenanceNotificationMode.Auto` exists, and why the default is `Disabled`. +2. **The wire contract is still moving.** Notification types have been proposed and withdrawn during + its development, the semantics of some fields (notably the sequence number) are not defined by any + specification, and no captured RESP transcript exists to validate against. Our parsing is + deliberately liberal, but the shapes may change. +3. **What the client *does* in response is the substantial part**, and it is being built in stages - + timeout relaxation, then endpoint handoff. Behaviour may therefore change materially between + versions while the diagnostic is in place, even where the API does not. + +`MaintenanceNotificationMode`: + +- `Disabled` (the default) - never ask; the server sends nothing. +- `Enabled` - **required**: ask, and *reject the connection* unless notifications end up live. That covers a + server that refuses, a server that answers `HELLO 3` as RESP2, and a configuration that could never ask in + the first place (`Protocol = Resp2`, or `HELLO` unavailable) - requiring a RESP3-only feature over RESP2 is + a contradiction, and it fails rather than being half-honoured. Only point this at a deployment you know + supports them. +- `Auto` - ask, and carry on if the server refuses or the connection ends up RESP2; the feature is then + simply off for that server and the connection is never rejected over it. Safe against a mixture of + servers, or one whose support you don't know, and what most callers want. + +The names are the cross-client ones (they match go-redis and redis-py, so a connection string ports +between clients), which makes `Enabled` easy to misread as "on" - it means "required". One deliberate +divergence: the cross-client spec only calls for interrupting the connection when the *server* errors, +whereas we extend that to the RESP2 cases above, on the grounds that a required feature which cannot +possibly be delivered is the same failure either way. + +## A deployment prerequisite worth knowing + +On Redis Enterprise there are *two* switches, and they are easy to conflate. A **cluster-level flag** decides +whether the subcommand exists at all - `client_maint_notifications` for proxy-routed databases, +`oss_cluster_client_maint_notifications` for `oss_cluster` ones - and the **per-connection opt-in** this option +controls asks one connection to receive them. A cluster running a supporting version with the flag off will +refuse `CLIENT MAINT_NOTIFICATIONS ON`, which `Auto` absorbs silently and `Enabled` turns into a refused +connection. So if `Enabled` is rejecting connections against a deployment you believe supports the feature, +check the cluster flag before suspecting the client. + +If you accept the above, you can suppress this warning by adding the following to your `csproj` file: + +```xml +$(NoWarn);SER010 +``` + +or more granularly / locally in C#: + +``` c# +#pragma warning disable SER010 +``` diff --git a/src/RESPite/Shared/Experiments.cs b/src/RESPite/Shared/Experiments.cs index 2cfcd94d6..7077164ad 100644 --- a/src/RESPite/Shared/Experiments.cs +++ b/src/RESPite/Shared/Experiments.cs @@ -21,6 +21,7 @@ internal static class Experiments public const string GeoRedundantFailover = "SER007"; public const string Server_8_10 = "SER008"; public const string Transport = "SER009"; + public const string MaintenanceNotifications = "SER010"; // ReSharper restore InconsistentNaming diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index d533d4506..e8b800521 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -39,11 +39,13 @@ public FaultContext(Exception fault) kind = RedisErrorKind.ConnectionFault; flags = connection.Flags; status = connection.CommandStatus; + MaintenanceType = connection.MaintenanceType; break; case RedisTimeoutException timeout: kind = RedisErrorKind.Timeout; flags = timeout.Flags; status = timeout.Commandstatus; + MaintenanceType = timeout.MaintenanceType; break; case TimeoutException: kind = RedisErrorKind.Timeout; @@ -118,6 +120,18 @@ public FaultContext(Exception fault) /// public ConnectionFailureType ConnectionFailureType => _connectionFailureType; + /// + /// The maintenance notification in force when this fault happened, if any. + /// + /// + /// Useful to a circuit breaker: a fault during announced maintenance is expected and transient, and + /// counting it towards a trip that then withdraws a whole server is the opposite of what the notification + /// was for. Left to policy rather than acted on here, since "ignore faults during maintenance" is a + /// judgement about the deployment, not about the protocol. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public Maintenance.MaintenanceNotificationType MaintenanceType { get; } + private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) { // the client never handed it to the socket, so the server cannot have seen it diff --git a/src/StackExchange.Redis/Availability/HealthCheckContext.cs b/src/StackExchange.Redis/Availability/HealthCheckContext.cs index 6d3fd5037..3322ef421 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckContext.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -27,4 +27,16 @@ public readonly struct HealthCheckContext(IServer server, TimeSpan probeTimeout) /// themselves (the caller applies it), but may use it to bound any state they create. /// public TimeSpan ProbeTimeout => probeTimeout; + + /// + /// Gets flags that a probe should include on every command it issues. + /// + /// + /// This marks the traffic as a health check rather than as work a caller is waiting on. It matters because + /// idleness is what decides whether an endpoint can be given up: a probe that looks like caller work makes + /// a server appear busy precisely because we are watching it, and an endpoint that has left the deployment + /// then never gets retired. The built-in probes pass this; a custom probe should too, and passing it + /// changes nothing else about how the command is routed or queued. + /// + public CommandFlags ProbeFlags => Message.ProbeFlag; } diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs index 64b1e882d..3e1491a6c 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs @@ -1,4 +1,4 @@ -using System.Threading.Tasks; +using System.Threading.Tasks; namespace StackExchange.Redis.Availability; @@ -16,7 +16,7 @@ private PingProbe() { } public override async Task CheckHealthAsync(HealthCheckContext context) { - await context.Server.PingAsync(); + await context.Server.PingAsync(context.ProbeFlags); return HealthCheckResult.Healthy; } } diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs index 6e1524b7a..058cff727 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Buffers; using System.Threading.Tasks; @@ -39,10 +39,10 @@ await database.LockTakeAsync( key: key, value: payload, expiry: context.ProbeTimeout, - flags: CommandFlags.FireAndForget).ForAwait(); + flags: CommandFlags.FireAndForget | context.ProbeFlags).ForAwait(); // release from the db if matches (otherwise, we have no clue what happened, so: leave alone) - var success = await database.LockReleaseAsync(key, payload).ForAwait(); + var success = await database.LockReleaseAsync(key, payload, context.ProbeFlags).ForAwait(); return success ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; } finally diff --git a/src/StackExchange.Redis/ClusterConfiguration.cs b/src/StackExchange.Redis/ClusterConfiguration.cs index 2880cf6ac..08fcab15e 100644 --- a/src/StackExchange.Redis/ClusterConfiguration.cs +++ b/src/StackExchange.Redis/ClusterConfiguration.cs @@ -137,21 +137,93 @@ public override int GetHashCode() internal bool Includes(int hashSlot) => hashSlot >= from && hashSlot <= to; - private static bool TryParseInt16(string s, int offset, int count, out short value) + /// + /// Parses one range from within a larger string, so that a comma-separated list can be read without + /// allocating a substring per element. + /// + internal static bool TryParse(string value, int offset, int length, out SlotRange range) { - checked + range = default; + if (length <= 0) return false; + + int dash = -1; + for (int i = 0; i < length; i++) + { + if (value[offset + i] == '-') + { + dash = i; + break; + } + } + + if (dash < 0) { - value = 0; - int tmp = 0; - for (int i = 0; i < count; i++) + if (TryParseInt16(value, offset, length, out var only)) { - char c = s[offset + i]; - if (c < '0' || c > '9') return false; - tmp = (tmp * 10) + (c - '0'); + range = new SlotRange(only, only); + return true; } - value = (short)tmp; + return false; + } + + if (dash != 0 && dash != length - 1 + && TryParseInt16(value, offset, dash, out var from) + && TryParseInt16(value, offset + dash + 1, length - dash - 1, out var to)) + { + // a reversed range is a server bug, not something to normalize silently + if (from > to) return false; + range = new SlotRange(from, to); return true; } + + return false; + } + + /// + /// Parses the comma-and-range slot form used by the maintenance notifications, e.g. + /// 123,456,789-1000. Empty elements are skipped; a malformed element fails the whole list, + /// since a partially-read slot set is worse than none. + /// + internal static bool TryParseList(string? value, out List ranges) + { + ranges = []; + if (string.IsNullOrEmpty(value)) return false; + + int start = 0; + while (start <= value!.Length) + { + var comma = value.IndexOf(',', start); + var length = (comma < 0 ? value.Length : comma) - start; + if (length > 0) + { + if (!TryParse(value, start, length, out var range)) return false; + ranges.Add(range); + } + + if (comma < 0) break; + start = comma + 1; + } + + return ranges.Count != 0; + } + + private static bool TryParseInt16(string s, int offset, int count, out short value) + { + // note this deliberately does not use `checked`: it used to, and an out-of-range slot number + // therefore threw OverflowException from a Try* method - reachable from the public + // SlotRange.TryParse and from CLUSTER NODES parsing, and now also from the maintenance + // notification reader, where throwing on the read loop is far worse than rejecting a value + value = 0; + int tmp = 0; + for (int i = 0; i < count; i++) + { + char c = s[offset + i]; + if (c < '0' || c > '9') return false; + tmp = (tmp * 10) + (c - '0'); + if (tmp > short.MaxValue) return false; // as soon as it cannot fit, stop + } + value = (short)tmp; + return true; } int IComparable.CompareTo(object? obj) => obj is SlotRange sRange ? CompareTo(sRange) : -1; diff --git a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs index 42bb2fcbf..51e2caacf 100644 --- a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs @@ -1,6 +1,8 @@ using System; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Threading.Tasks; +using RESPite; namespace StackExchange.Redis.Configuration { @@ -29,6 +31,9 @@ public class AzureManagedRedisOptionsProvider : DefaultOptionsProvider ".redisenterprise.cache.azure.net", ]; + /// + public override string Name => "amr"; + /// public override bool IsMatch(EndPoint endpoint) { @@ -65,5 +70,18 @@ public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action public override string ConfigurationChannel => ""; // disable on AMR + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// Pre-emptive: AMR does not emit these yet, and support is being added concurrently with this + /// client-side work. is what makes that safe - until + /// the server side ships, the opt-in is refused and the feature stays off, and it then starts working + /// without anybody needing to change a connection string. AMR also already prefers RESP3 here, which + /// the feature requires. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; } } diff --git a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs index c02f8f760..ee577eb59 100644 --- a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs @@ -33,6 +33,9 @@ public class AzureOptionsProvider : DefaultOptionsProvider ".redis.cache.sovcloud-api.fr", }; + /// + public override string Name => "azure"; + /// public override bool IsMatch(EndPoint endpoint) { diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index afcce011f..a278f435f 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using RESPite; namespace StackExchange.Redis.Configuration { @@ -27,6 +29,10 @@ public class DefaultOptionsProvider { new AzureOptionsProvider(), new AzureManagedRedisOptionsProvider(), + new RedisCloudOptionsProvider(), + + // matches nothing, so its position is irrelevant; it is here to be resolvable by name + new RedisEnterpriseOptionsProvider(), }; /// @@ -51,6 +57,68 @@ public static void AddProvider(DefaultOptionsProvider provider) /// public virtual bool IsMatch(EndPoint endpoint) => false; + /// + /// The name this provider can be selected by in a configuration string, as defaults={name}; + /// null (the default) means it cannot be named, and can only be selected in code or by + /// . + /// + /// + /// This exists for the deployments cannot recognize: an on-premise + /// Enterprise cluster has arbitrary DNS, and a hosted one reached through private DNS or a proxy no + /// longer looks like itself. Naming is also what makes a provider expressible in the connection string + /// an application is configured with, rather than only in code it would have to be rebuilt to change. + /// + /// Only providers registered with or built in can be + /// resolved by name - deliberately, since a configuration string that could name an arbitrary type + /// would be a way to have one loaded, and would defeat trimming. + /// + /// + public virtual string? Name => null; + + /// + /// The provider's where it has one, and the type name otherwise. + /// + /// + /// Display only - a provider in a log should read as amr rather than as a namespace-qualified + /// type name. Note that serialization tests directly rather than going through + /// here: this never returns null, and an unnameable provider must not end up in a + /// configuration string. + /// + public override string ToString() => Name ?? base.ToString()!; + + /// + /// Finds a registered provider by its . + /// + internal static bool TryGetByName(string name, [NotNullWhen(true)] out DefaultOptionsProvider? provider) + { + foreach (var candidate in KnownProviders) + { + if (candidate.Name is { } candidateName && string.Equals(candidateName, name, StringComparison.OrdinalIgnoreCase)) + { + provider = candidate; + return true; + } + } + + provider = null; + return false; + } + + /// + /// The names that defaults= accepts, for diagnostics. + /// + internal static string GetKnownNames() + { + var names = new List(); + foreach (var candidate in KnownProviders) + { + if (candidate.Name is { } name && !names.Contains(name)) names.Add(name); + } + + names.Sort(StringComparer.OrdinalIgnoreCase); + return string.Join(", ", names); + } + /// /// Gets a provider for the given endpoints, falling back to if nothing more specific is found. /// @@ -267,6 +335,50 @@ protected virtual string GetDefaultClientName() => /// public virtual RedisProtocol? Protocol => null; + /// + /// Gets whether to ask servers to send maintenance notifications. + /// + /// + /// here on purpose. The notification contract asks + /// clients to default to auto, and that is right for the deployments that emit them - but this + /// library is pointed at every RESP implementation there is, most of which have never heard of the + /// opt-in, and asking them all costs a handshake slot plus an error reply in their logs. Nor is the + /// server the only thing in the path: proxies sit in front of these deployments, and an unrecognized + /// CLIENT subcommand is not guaranteed to be answered as politely as a server would. A provider + /// that recognizes a deployment which supports the feature should override this. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Disabled; + + /// + /// Gets the value command timeouts are relaxed to during an announced disruption; 10 seconds, as the + /// notification contract prescribes. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan MaintenanceRelaxedTimeout => TimeSpan.FromSeconds(10); + + /// + /// Gets the longest a relaxed window may last, or null to derive it as three times the + /// effective . + /// + /// + /// Deriving is the default because the two have to stay coherent: a cap below the timeout it bounds is + /// nonsense, and it is the *configured* relaxed timeout that matters, which this type cannot see. + /// At the prescribed 10 seconds that gives 30 - the contract caps the MOVING budget at 15, so + /// there is ample room for a legitimate window while a stuck one is bounded to half a minute. Return a + /// value here to pin it regardless of the relaxed timeout. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan? MaintenanceRelaxedWindowMax => null; + + /// + /// Gets how long timeouts stay relaxed after a disruption reports completion, or null to derive + /// it as twice the effective + /// (which matches go-redis at the prescribed default). + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan? MaintenancePostEventRelaxedDuration => null; + /// /// Gets whether to enable TCP keep-alive when appropriate (endpoint- and platform-dependent). /// diff --git a/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs b/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs new file mode 100644 index 000000000..fc00c4f9b --- /dev/null +++ b/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs @@ -0,0 +1,83 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Configuration +{ + /// + /// Options provider for Redis Cloud environments. + /// + /// + /// Deliberately *not* a copy of , despite the deployments + /// looking similar: AMR is TLS-only and has a 7.4 floor, and Redis Cloud is neither. See the individual + /// members for what is shared and what is not. + /// + public class RedisCloudOptionsProvider : DefaultOptionsProvider + { + // note the third subsumes the first (EndsWith), so it is kept for the documentation rather than the + // logic - the narrower entry records what the common case actually looks like + private static readonly string[] redisCloudDomains = + [ + ".cloud.redislabs.com", // standard fully-managed endpoints (AWS, GCP, Azure BYOC) + ".cloud.redis.io", // newer routing scheme for managed instances + ".redislabs.com", // older subscriptions, addressed directly + ]; + + /// + public override string Name => "rediscloud"; + + /// + public override bool IsMatch(EndPoint endpoint) + => endpoint is DnsEndPoint dnsEp && IsHostInDomains(dnsEp.Host, redisCloudDomains); + + private static bool IsHostInDomains(string hostName, string[] domains) + { + foreach (var domain in domains) + { + if (hostName.EndsWith(domain, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } + + /// + /// Allow connecting after startup, in the cases where the remote cache isn't ready or is overloaded. + /// + public override bool AbortOnConnectFail => false; + + /// + /// Prefer RESP3, which the deployment supports and which maintenance notifications require. + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; + + /// + /// Disabled: the deployment is proxied, so the OSS configuration-broadcast channel conveys nothing. + /// + public override string ConfigurationChannel => ""; + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// This is the deployment family the feature exists for. + /// rather than because a database that has not been + /// updated yet must keep working: the opt-in is then refused and the feature stays off, rather than the + /// connection being rejected. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; + + // Two things AzureManagedRedisOptionsProvider does that are deliberately *not* repeated here: + // + // - GetDefaultSsl => true. AMR is TLS-only, so assuming TLS there is safe. Redis Cloud enables TLS + // per database and plenty of databases are plaintext, so defaulting it on would fail their connect + // outright - a much worse outcome than not having guessed. + // - DefaultVersion => 7.4. That is AMR's floor. Redis Cloud still offers older versions per database, + // and claiming a version we do not have unlocks commands the server will reject, so the base + // default stands. + } +} diff --git a/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs b/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs new file mode 100644 index 000000000..87634fbf5 --- /dev/null +++ b/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Configuration +{ + /// + /// Options provider for a self-managed Redis Enterprise deployment, selected explicitly. + /// + /// + /// Deliberately matches no endpoint. An on-premise cluster has whatever DNS its operator gave it, so there + /// is nothing to recognize - which is exactly why this provider is nameable: it can be asked for as + /// defaults=enterprise in a configuration string, or assigned to + /// in code. + /// + /// It is also the right choice for a hosted deployment reached somewhere its own provider cannot see it - + /// behind private DNS, a CNAME, or a proxy - where the endpoint no longer looks like what it is. + /// + /// + public class RedisEnterpriseOptionsProvider : DefaultOptionsProvider + { + /// + public override string Name => "enterprise"; + + /// + /// Prefer RESP3, which the deployment supports and which maintenance notifications require. + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; + + /// + /// Disabled: the deployment is proxied, so the OSS configuration-broadcast channel conveys nothing. + /// + public override string ConfigurationChannel => ""; + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// rather than + /// : a cluster that has not been updated yet, or has + /// the feature switched off, must keep working. Choose Enabled explicitly if you would rather a + /// connection be refused than run without advance warning. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; + + // Note: no GetDefaultSsl and no DefaultVersion override. Both are deployment choices here rather than + // properties of the product - TLS is configured per database, and the version is whatever was + // installed - so guessing either would be worse than leaving the library's defaults in place. + } +} diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 12db0bac1..94b95f7c0 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -73,6 +73,49 @@ internal static Proxy ParseProxy(string key, string value) return tmp; } + internal static DefaultOptionsProvider ParseDefaultsProvider(string key, string value) + { + if (!DefaultOptionsProvider.TryGetByName(value, out var provider)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a known defaults provider name; '{value}' is not one of: {DefaultOptionsProvider.GetKnownNames()}."); + } + return provider; + } + + internal static MaintenanceNotificationMode ParseMaintenanceNotifications(string key, string value) + { + if (!Enum.TryParse(value, true, out MaintenanceNotificationMode tmp) || !Enum.IsDefined(typeof(MaintenanceNotificationMode), tmp)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a MaintenanceNotificationMode value; the value '{value}' is not recognised."); + } + return tmp; + } + + /// + /// Parses one of the maintenance durations, which are expressed in seconds - the unit the + /// cross-client contract uses for maintRelaxedTimeout, so a documented value can be pasted + /// between clients. + /// + /// + /// Seconds is the odd one out in this file, where every other timeout is milliseconds, and the + /// mistake is silent in one direction: a caller who assumes milliseconds and writes 30000 would + /// otherwise get an eight-hour relaxed timeout. Hence the upper bound, whose message names the + /// unit - it exists to turn that into a diagnosable error rather than a mystery. + /// + internal static TimeSpan ParseMaintenanceSeconds(string key, string value) + { + const int MaxSeconds = 600; + if (!Format.TryParseInt32(value, out int seconds) || seconds < 0) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a non-negative integer number of seconds; the value '{value}' is not valid."); + } + if (seconds > MaxSeconds) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' is expressed in seconds, and {seconds}s exceeds the maximum of {MaxSeconds}s; note that this option is not in milliseconds."); + } + return TimeSpan.FromSeconds(seconds); + } + internal static SslProtocols ParseSslProtocols(string key, string? value) { // Flags expect commas as separators, but we need to use '|' since commas are already used in the connection string to mean something else @@ -125,6 +168,11 @@ internal const string Tunnel = "tunnel", SetClientLibrary = "setlib", Protocol = "protocol", + Defaults = "defaults", + MaintenanceNotifications = "maintNotifications", + MaintenanceRelaxedTimeout = "maintRelaxedTimeout", + MaintenanceRelaxedWindowMax = "maintRelaxedWindowMax", + MaintenancePostEventRelaxedDuration = "maintPostEventRelaxed", HighIntegrity = "highIntegrity", TcpKeepAlive = "tcpKeepAlive"; @@ -162,6 +210,11 @@ internal const string Tunnel, SetClientLibrary, Protocol, + Defaults, + MaintenanceNotifications, + MaintenanceRelaxedTimeout, + MaintenanceRelaxedWindowMax, + MaintenancePostEventRelaxedDuration, HighIntegrity, TcpKeepAlive, }.ToDictionary(x => x, StringComparer.OrdinalIgnoreCase); @@ -217,6 +270,11 @@ private enum OptionFlags : ulong SslProtocolsHasValue = 1UL << 32, ProtocolHasValue = 1UL << 33, AllowSimulateConnectionFailure = 1UL << 34, + MaintenanceNotificationsHasValue = 1UL << 35, + DefaultsHasValue = 1UL << 36, + MaintenanceRelaxedTimeoutHasValue = 1UL << 37, + MaintenanceRelaxedWindowMaxHasValue = 1UL << 38, + MaintenancePostEventRelaxedDurationHasValue = 1UL << 39, } private OptionFlags optionFlags; @@ -242,6 +300,8 @@ private enum OptionFlags : ulong private SslProtocols sslProtocols; private RedisProtocol _protocol; + private MaintenanceNotificationMode _maintenanceNotifications; + private TimeSpan _maintenanceRelaxedTimeout, _maintenanceRelaxedWindowMax, _maintenancePostEventRelaxedDuration; private bool HasValue(OptionFlags hasValue) => (optionFlags & hasValue) != 0; @@ -314,7 +374,15 @@ private void SetWithValue(OptionFlags hasValue, ref T storage, T value) where public DefaultOptionsProvider Defaults { get => defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); - set => defaultOptions = value; + set + { + defaultOptions = value; + + // the getter memoizes an *inferred* provider into the same field, so a flag is the only way to + // tell "the caller chose this" from "we worked it out from the endpoints" - and only the former + // may be written back out to a configuration string + optionFlags |= OptionFlags.DefaultsHasValue; + } } /// @@ -1004,6 +1072,10 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow Tunnel = Tunnel, LibraryName = LibraryName, _protocol = _protocol, + _maintenanceNotifications = _maintenanceNotifications, + _maintenanceRelaxedTimeout = _maintenanceRelaxedTimeout, + _maintenanceRelaxedWindowMax = _maintenanceRelaxedWindowMax, + _maintenancePostEventRelaxedDuration = _maintenancePostEventRelaxedDuration, heartbeatInterval = heartbeatInterval, WriteMode = WriteMode, CircuitBreaker = CircuitBreaker, @@ -1097,6 +1169,13 @@ public string ToString(bool includePassword) Append(sb, OptionKeys.SetClientLibrary, OptionFlags.SetClientLibraryHasValue, OptionFlags.SetClientLibraryValue); Append(sb, OptionKeys.HighIntegrity, OptionFlags.HighIntegrityHasValue, OptionFlags.HighIntegrityValue); if (HasValue(OptionFlags.ProtocolHasValue)) Append(sb, OptionKeys.Protocol, FormatProtocol(_protocol)); + // only when the caller set it *and* it can be named: an inferred provider must not be baked into + // the string, or re-parsing would pin a choice that was only ever a guess from the endpoints + if (HasValue(OptionFlags.DefaultsHasValue) && defaultOptions?.Name is { } defaultsName) Append(sb, OptionKeys.Defaults, defaultsName); + if (HasValue(OptionFlags.MaintenanceNotificationsHasValue)) Append(sb, OptionKeys.MaintenanceNotifications, _maintenanceNotifications.ToString()); + if (HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedTimeout, FormatMaintenanceSeconds(_maintenanceRelaxedTimeout)); + if (HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedWindowMax, FormatMaintenanceSeconds(_maintenanceRelaxedWindowMax)); + if (HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue)) Append(sb, OptionKeys.MaintenancePostEventRelaxedDuration, FormatMaintenanceSeconds(_maintenancePostEventRelaxedDuration)); Append(sb, OptionKeys.TcpKeepAlive, OptionFlags.TcpKeepAliveHasValue, OptionFlags.TcpKeepAliveValue); if (Tunnel is { IsInbuilt: true } tunnel) { @@ -1212,6 +1291,8 @@ private void Clear() #endif Tunnel = null; _protocol = default; + _maintenanceNotifications = default; + _maintenanceRelaxedTimeout = _maintenanceRelaxedWindowMax = _maintenancePostEventRelaxedDuration = default; WriteMode = default; CircuitBreaker = null; RetryPolicy = null; @@ -1368,6 +1449,21 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) case OptionKeys.Protocol: SetWithValue(OptionFlags.ProtocolHasValue, ref _protocol, OptionKeys.ParseRedisProtocol(key, value)); break; + case OptionKeys.Defaults: + Defaults = OptionKeys.ParseDefaultsProvider(key, value); + break; + case OptionKeys.MaintenanceNotifications: + SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, OptionKeys.ParseMaintenanceNotifications(key, value)); + break; + case OptionKeys.MaintenanceRelaxedTimeout: + SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; + case OptionKeys.MaintenanceRelaxedWindowMax: + SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; + case OptionKeys.MaintenancePostEventRelaxedDuration: + SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; // Deprecated options we ignore... case OptionKeys.HighPrioritySocketThreads: case OptionKeys.PreserveAsyncOrder: @@ -1420,6 +1516,86 @@ public RedisProtocol? Protocol set => Set(OptionFlags.ProtocolHasValue, ref _protocol, value); } + /// + /// Whether to ask servers to send maintenance notifications; note that + /// means required and rejects connections + /// that cannot deliver them - is the best-effort + /// mode. + /// + /// + /// Requires RESP3, and only Redis Enterprise and Redis Cloud emit them - so the default is + /// rather than spending an extra handshake command + /// asking every server in existence a question almost none of them understand. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationMode MaintenanceNotifications + { + get => HasValue(OptionFlags.MaintenanceNotificationsHasValue) ? _maintenanceNotifications : Defaults.MaintenanceNotifications; + set => SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, value); + } + + /// + /// The value command timeouts are relaxed to while a server has announced a disruption. + /// + /// + /// This is a floor, never a reduction: the effective timeout inside a window is + /// max(configured, this), so a caller with a generous timeout keeps it. Expressed in seconds in + /// a configuration string (maintRelaxedTimeout=10), matching the cross-client contract - note + /// that this is unlike every other timeout here, which are milliseconds. Only command timeouts are + /// relaxed; keep-alive, the heartbeat and connection-failure detection are deliberately untouched, so + /// a server that dies mid-maintenance is still noticed on the usual schedule. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenanceRelaxedTimeout + { + get => HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue) ? _maintenanceRelaxedTimeout : Defaults.MaintenanceRelaxedTimeout; + set => SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, value); + } + + /// + /// The longest a relaxed window may last, however long the server said the disruption would take. + /// + /// + /// A backstop for a closing notification that never arrives, and our invention - the + /// notification contract names no upper bound. A window that never closes is worse than one that + /// closes early, since relaxation delays the point at which a genuinely slow server surfaces as a + /// timeout. Expressed in seconds in a configuration string. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenanceRelaxedWindowMax + { + get => HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue) + ? _maintenanceRelaxedWindowMax + : Defaults.MaintenanceRelaxedWindowMax ?? Multiply(MaintenanceRelaxedTimeout, 3); + set => SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + } + + /// + /// How long to keep timeouts relaxed after a disruption reports that it has finished. + /// + /// + /// Also our invention. A closing notification means the server-side operation completed, not + /// that the server is back to normal latency - and the moment after it completes is precisely when + /// every other client that received the same notification re-engages, so the load spike arrives + /// slightly after the all-clear. Does not apply when a window ended by hitting + /// : in that case nothing told us the event finished, and + /// extending past the backstop would defeat it. Expressed in seconds in a configuration string. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenancePostEventRelaxedDuration + { + get => HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue) + ? _maintenancePostEventRelaxedDuration + : Defaults.MaintenancePostEventRelaxedDuration ?? Multiply(MaintenanceRelaxedTimeout, 2); + set => SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + } + + // TimeSpan * int is netstandard2.1+, so this is ticks arithmetic to keep the down-level TFMs building + private static TimeSpan Multiply(TimeSpan value, int factor) => TimeSpan.FromTicks(value.Ticks * factor); + + private static string FormatMaintenanceSeconds(TimeSpan value) + => ((int)value.TotalSeconds).ToString(System.Globalization.CultureInfo.InvariantCulture); + internal BufferedStreamWriter.WriteMode WriteMode { get; set; } /// diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs index 0a8b95be5..fd5bd7c7f 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs @@ -88,6 +88,69 @@ private void OnEndpointChanged(EndPoint endpoint, EventHandler public event EventHandler? ServerMaintenanceEvent; + + /// + /// How host names are resolved during a maintenance handoff. + /// + /// + /// A seam rather than a call to directly, because no in-process fake can move a + /// DNS record: the handoff's whole decision turns on the answer *changing*, and the only way to test that + /// deterministically is to supply the answers. Defaults to real DNS. + /// + internal Func> AddressResolver { get; set; } + = Maintenance.AdvertisedAddressProbe.DefaultResolveAsync; + + // recently-raised (type, sequence) pairs, so one logical event raises one event however many nodes told + // us. Small and fixed: the copies arrive within milliseconds of each other, so a handful of slots covers + // any realistic proxy count even with other notifications interleaved. + // A named struct rather than a tuple: this assembly must not reference System.ValueTuple, which breaks + // .NET Framework consumers - see SanityChecks.ValueTupleNotReferenced. + private readonly struct RaisedMaintenanceEvent(Maintenance.MaintenanceNotificationType type, long sequence) + { + public readonly Maintenance.MaintenanceNotificationType Type = type; + public readonly long Sequence = sequence; + } + + private readonly RaisedMaintenanceEvent[] _raisedMaintenanceEvents = new RaisedMaintenanceEvent[8]; + private int _raisedMaintenanceEventIndex; + + /// + /// Whether this is the first time we have been told about a given maintenance event, across every + /// connection. + /// + /// + /// Every node broadcasts a given event, and all of them carry the same sequence number - observed on + /// Enterprise 8.6.2, where the id identifies the event rather than the delivery. So without this, a + /// deployment fronted by three proxies raises three events for one migration and every consumer has to + /// dedupe them. + /// + /// Matched on equality rather than "less than or equal", deliberately: a lagging node reporting an + /// *earlier* event we have not seen yet is a distinct event and must still be raised. Only an exact repeat + /// of something already raised is a duplicate. + /// + /// + /// Eviction is the only expiry - an entry falls out once eight further notifications have been recorded, so + /// nothing has to be purged on a timer and the state cannot grow. A duplicate arriving after its entry has + /// been evicted would be raised a second time, which is the right way round to be wrong: the copies arrive + /// within milliseconds of each other, so that takes a straggler behind eight intervening events. + /// + /// + internal bool TryClaimMaintenanceEvent(Maintenance.MaintenanceNotificationType type, long? sequence) + { + if (sequence is not { } seq) return true; // no id to match on; better a duplicate than a silence + + lock (_raisedMaintenanceEvents) + { + foreach (var entry in _raisedMaintenanceEvents) + { + if (entry.Type == type && entry.Sequence == seq) return false; + } + + _raisedMaintenanceEvents[_raisedMaintenanceEventIndex] = new RaisedMaintenanceEvent(type, seq); + _raisedMaintenanceEventIndex = (_raisedMaintenanceEventIndex + 1) % _raisedMaintenanceEvents.Length; + return true; + } + } internal void OnServerMaintenanceEvent(ServerMaintenanceEvent e) => ServerMaintenanceEvent?.Invoke(this, e); diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 903941c1f..d4182cf41 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2492,7 +2492,25 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u throw GetException(result, message, server); } - if (Monitor.Wait(source, TimeoutMilliseconds)) + // Unlike the async sweeps, which re-read the effective timeout on every heartbeat, a + // sync caller commits to a duration when it parks - so if maintenance relaxation + // begins while we are waiting, we have to notice by re-waiting rather than failing at + // the original deadline. Without this a sync caller in flight when a MIGRATING + // arrives times out at the strict timeout while its async neighbour is relaxed. + var watch = ValueStopwatch.StartNew(); + bool completed = Monitor.Wait(source, server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds); + while (!completed) + { + var elapsed = watch.ElapsedMilliseconds; + var revised = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds; + + // also the path back: if a window closed while we waited, revised drops to the + // configured value and we stop + if (revised <= elapsed) break; + completed = Monitor.Wait(source, revised - elapsed); + } + + if (completed) { Trace("Timely response to " + message); } diff --git a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs index 9f0df5ba0..3fd7847b3 100644 --- a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs +++ b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs @@ -68,5 +68,22 @@ public enum ConnectionFailureType /// [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] CircuitBreaker, + + /// + /// The connection was replaced deliberately, to move off an endpoint the server announced it is + /// retiring. + /// + /// + /// Not a fault: the connection was working, and we chose to replace it while a replacement address was + /// known rather than wait to be disconnected. Reported here because the alternative is worse - the + /// replacement raises , so without this a consumer + /// tracking connection state sees a restore with no matching failure, and no reason for the churn. + /// + /// Consumers alerting on should filter this out: + /// it is expected during planned maintenance and says nothing is wrong. is + /// the precedent for a deliberate client action being reported this way. + /// + /// + MaintenanceHandoff, } } diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index aa9bf7001..c2977de0e 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -319,14 +319,19 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; + // if the server had announced a disruption, say so on the fault: "timeout" and "timeout during an + // announced failover" call for very different reactions from whoever reads the log + var maintenanceType = server?.ActiveMaintenanceType ?? Maintenance.MaintenanceNotificationType.None; Exception ex = logConnectionException && lastConnectionException is not null ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, + MaintenanceType = maintenanceType, } : new RedisTimeoutException(msgFlags, sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, + MaintenanceType = maintenanceType, }; CopyDataToException(data, ex); diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 03c62ed95..7febd6dde 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Text; using System.Threading.Tasks; @@ -765,4 +765,10 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) EventId = 110, Message = "{BridgeName}: Transport connected (encrypted: {IsEncrypted})")] internal static partial void LogInformationTransportConnected(this ILogger logger, string bridgeName, bool isEncrypted); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 116, + Message = "{Server}: Requesting maintenance notifications ({Mode})")] + internal static partial void LogInformationRequestingMaintenanceNotifications(this ILogger logger, ServerEndPointLogValue server, MaintenanceNotificationMode mode); } diff --git a/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs b/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs new file mode 100644 index 000000000..6170e4dc1 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs @@ -0,0 +1,227 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Maintenance; + +/// +/// Asks DNS the two questions a handoff needs: what replaces the address we are leaving, and are we still +/// advertised at all. +/// +/// +/// A poll rather than a lookup, and that is the whole point. Measured on Redis Enterprise (2026-08-28), with +/// times relative to the notification: the server-side endpoint moves at +8.6s, DNS follows at +9.7s and +4.4s +/// across two runs, and the sockets close at +18.4s and +16.6s - against a declared 15s grace and a 5s record +/// TTL. So resolving immediately returns the address being retired, in every run observed, and a +/// client that treats the first answer as authoritative hands off to the node it was told to leave. +/// +/// DNS is not guaranteed to win the race. On a second cluster the same notification with the same 15s +/// grace saw the socket close at +15.7s and DNS update at +18.7s - three seconds after the close. So +/// there are three outcomes here, not two, and the third is a normal one: the record may still be stale when +/// the window runs out, and the only thing left is to reconnect after the close and resolve then. Do not +/// "simplify" the null return away. +/// +/// +/// There is no way to observe the intermediate state from outside: the endpoint has moved server-side well +/// before DNS reflects it, and nothing tells a client which of those has happened. Probing until the answer +/// changes is the only mechanism available, and the short TTL is what makes it work - several attempts fit +/// inside the window. +/// +/// +/// The rule is "take any address that is not the one being retired". Note it is deliberately *not* "prefer an +/// address that has newly appeared", even though a MOVING implies one exists - it is emitted when the +/// connection's own proxy *leaves* the endpoint's address set **and** the set gains a member (established +/// across thirteen observations: a pure reduction takes the proxy away without announcing anything, and a pure +/// widening adds addresses while leaving the connection's proxy in place, which is equally silent). A live +/// sibling is at least as good a target as a newly joined node and is available *now*, whereas the new node +/// only becomes visible when the record updates - which can be after the socket has already closed. Preferring +/// the newcomer would mean waiting for it, and waiting is the thing to avoid. The one case where it would pay +/// is a rolling operation, where the sibling we step to may take its own turn later; that costs one further +/// handoff, bounded at one per connection per operation, which is cheaper than a lost window. +/// +/// +/// This rule is also doing more work than it looks. A Redis Cloud hostname usually carries several A records - measured 2026-08-28: 2 for +/// all-nodes, 3 for all-master-shards, 1 for single, all on a 5s TTL - and the count +/// follows actual proxy *placement* rather than the policy name, so a multi-proxy database whose shards happen +/// to share a node still resolves to one address. With several records the first resolution already names a +/// live sibling proxy, so this returns immediately and steps sideways rather than waiting: any proxy of the +/// same database serves the same data, so a sibling now beats the replacement in nine seconds. The poll only +/// engages when the record names nothing but the address being retired - the single-address case, which is +/// exactly where waiting is the only option available. +/// +/// +/// Deliberately free of jitter, clocks-by-configuration and connection state, so that it can be tested +/// exhaustively without a server: the caller supplies the resolver, the interval and the budget. Jitter +/// belongs at the call site, where the existing refresh jitter lives. +/// +/// +internal static class AdvertisedAddressProbe +{ + /// + /// Ordinary DNS, which is what everything but a test uses. + /// + internal static Task DefaultResolveAsync(string host, CancellationToken cancellationToken) => +#if NET + Dns.GetHostAddressesAsync(host, cancellationToken); +#else + Dns.GetHostAddressesAsync(host); +#endif + + /// + /// Polls DNS until it stops naming , or until the window runs out. + /// + /// + /// The replacement endpoint, or null if the window expired without DNS moving - a measured outcome + /// rather than a failure (see the type remarks: DNS has been seen updating three seconds after the socket + /// closed). The caller should then do nothing proactive: the server closes the socket, the reconnect that + /// follows re-resolves anyway because the endpoint is a hostname, and the relaxed timeout window covers + /// the gap. Guessing an address here would be strictly worse. + /// + internal static async Task ProbeAsync( + DnsEndPoint endpoint, + IPAddress retiring, + TimeSpan window, + TimeSpan pollInterval, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (retiring is null) throw new ArgumentNullException(nameof(retiring)); + if (resolve is null) throw new ArgumentNullException(nameof(resolve)); + + // A non-positive window is not an error: a notification can arrive with nothing left of its budget + // (the shard notifications legitimately carry zero or negative times), and "act now" means one attempt + // rather than none. + var deadline = Environment.TickCount + (int)Math.Max(window.TotalMilliseconds, 0); + int attempt = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + attempt++; + + IPAddress[]? addresses = null; + try + { + addresses = await resolve(endpoint.Host, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // a DNS blip mid-handoff is exactly when we can least afford to give up; keep trying until + // the window says otherwise + log?.Invoke($"MOVING: resolve attempt {attempt} for {endpoint.Host} failed: {ex.Message}"); + } + + if (addresses is not null) + { + IPAddress? candidate = null; + bool retiringStillAdvertised = false; + foreach (var address in addresses) + { + if (address.Equals(retiring)) retiringStillAdvertised = true; + else candidate ??= address; + } + + if (candidate is not null) + { + // Two operationally different outcomes, worth distinguishing in a log somebody reads while + // debugging a handoff: we either stepped sideways to a proxy that was already there, or the + // record itself has moved on to the replacement. + log?.Invoke(retiringStillAdvertised + ? $"MOVING: {endpoint.Host} still advertises {retiring}; moving to sibling {candidate} (attempt {attempt})" + : $"MOVING: {endpoint.Host} now resolves to {candidate} after {attempt} attempt(s)"); + return new IPEndPoint(candidate, endpoint.Port); + } + + log?.Invoke($"MOVING: {endpoint.Host} still resolves only to {retiring} (attempt {attempt}); not yet updated"); + } + + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining <= 0) + { + log?.Invoke($"MOVING: {endpoint.Host} never stopped resolving to {retiring} within the window"); + return null; + } + + // never sleep past the deadline: the last attempt should land inside the window, not after it + var delay = (int)Math.Min(pollInterval.TotalMilliseconds, remaining); + if (delay > 0) await Task.Delay(delay, cancellationToken).ForAwait(); + } + } + + /// + /// Whether the address we are connected on is still one of the addresses the hostname advertises. + /// + /// + /// true if still advertised, false if the record no longer names it, and null if DNS + /// could not be asked - which is *not* the same as "no": a resolution failure is no reason to give up a + /// working connection. + /// + /// + /// The trigger that needs no notification, and the reason it matters is a measured gap. On a multi-proxy + /// database, taking a node out for maintenance announced only the data-movement pair - MIGRATING at + /// +4.3s and MIGRATED at +16.6s, to every proxy - and then dropped the victim from DNS at +21.4s and + /// closed its socket *silently* at +34.7s, while sibling connections stayed up past +90s. No + /// MOVING was ever sent. + /// + /// So for thirteen seconds the condition was plainly visible to anybody who asked - our address is no + /// longer advertised - and the client's only other signal was the socket dying with no explanation. Asking + /// this question when a MIGRATED arrives converts that into a controlled handoff. Note + /// MIGRATED is also the notification the server *retains* and replays on connect, so a client that + /// arrives mid-operation gets the same prompt. + /// + /// + /// The same condition is what a support case turned on: a client kept dialling endpoints that no longer + /// existed for 37 hours, because nothing it could observe told it to stop. Two unrelated failures, one + /// detection rule. + /// + /// + internal static async Task IsStillAdvertisedAsync( + DnsEndPoint endpoint, + IPAddress current, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (current is null) throw new ArgumentNullException(nameof(current)); + if (resolve is null) throw new ArgumentNullException(nameof(resolve)); + + IPAddress[] addresses; + try + { + addresses = await resolve(endpoint.Host, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + log?.Invoke($"{endpoint.Host}: cannot tell whether {current} is still advertised: {ex.Message}"); + return null; + } + + // an empty answer is "cannot tell" rather than "no": a record that momentarily resolves to nothing is + // a DNS problem, not an instruction to abandon a connection that is working + if (addresses is null || addresses.Length == 0) + { + log?.Invoke($"{endpoint.Host}: resolved to nothing; treating {current} as still advertised"); + return null; + } + + foreach (var address in addresses) + { + if (address.Equals(current)) return true; + } + + log?.Invoke($"{endpoint.Host}: no longer advertises {current}; it is being taken out of service"); + return false; + } +} diff --git a/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs b/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs new file mode 100644 index 000000000..7a9b7d950 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// One entry from a cluster slot-migration notification: some slots have moved, or are moving, from one node +/// to another. +/// +/// +/// A single SMIGRATED describes several of these at once, and not necessarily any involving the node +/// that sent it - every node in the cluster reports the same movements, so the sender is not implicitly the +/// source. +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public readonly struct ClusterSlotMigration +{ + internal ClusterSlotMigration(EndPoint? source, EndPoint? target, IReadOnlyList slots, string? raw) + { + Source = source; + Target = target; + Slots = slots; + RawSlots = raw; + } + + /// + /// The node the slots are moving from, or null if the server named it in a form that cannot be + /// dialled. + /// + public EndPoint? Source { get; } + + /// + /// The node the slots are moving to, or null if the server named it in a form that cannot be + /// dialled. + /// + public EndPoint? Target { get; } + + /// + /// The slots that are moving. + /// + public IReadOnlyList Slots { get; } + + /// + /// The slots exactly as the server expressed them, for the cases the parsed form does not survive - a + /// malformed list leaves empty and this populated. + /// + public string? RawSlots { get; } + + /// + public override string ToString() => $"{Format.ToString(Source)} -> {Format.ToString(Target)}: {RawSlots}"; +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs new file mode 100644 index 000000000..57959cdce --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs @@ -0,0 +1,47 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +/// +/// Carries maintenance context on the faults a disruption can cause, so that a timeout during an announced +/// migration is distinguishable from an ordinary one. +/// +/// +/// This follows the established pattern on these types - Commandstatus and Flags on +/// , FailureType on - rather +/// than introducing an exception type nobody is catching yet. Both properties are named for the role they +/// play, not for the type, matching FailureType. +/// +public sealed partial class RedisTimeoutException +{ + /// + /// The maintenance notification in force when this timed out, or + /// if the server had not announced anything. + /// + /// + /// A value here says the server told us to expect disruption, and that the timeout you are looking at + /// happened inside that window - including the tail after the disruption reported completion. It does not + /// promise that the maintenance *caused* the timeout, only that the two coincided; that is still the most + /// useful thing to know when reading a log after the fact. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationType MaintenanceType { get; internal init; } +} + +/// +public sealed partial class RedisConnectionException +{ + /// + /// The maintenance notification in force when this connection faulted, or + /// if the server had not announced anything. + /// + /// + /// Present on this type as well as on because a handoff that misses + /// its deadline surfaces either way round: as a timeout when the command was already in flight, and as a + /// connection fault when the endpoint went away underneath it. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationType MaintenanceType { get; internal init; } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs new file mode 100644 index 000000000..38df5c310 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs @@ -0,0 +1,133 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Maintenance; + +/// +/// What to do about a MOVING notification. +/// +internal enum HandoffAction +{ + /// Nothing useful is available; let the server close the socket and reconnect then. + None, + + /// Drop our connections so they re-establish against the replacement address. + Recycle, + + /// The replacement is a different endpoint; the topology has to be re-read to find it. + Reconfigure, +} + +/// +/// The outcome of deciding how to hand off, and why - the reason is logged either way. +/// +internal readonly struct HandoffDecision(HandoffAction action, EndPoint? target, string reason) +{ + public HandoffAction Action { get; } = action; + + public EndPoint? Target { get; } = target; + + public string Reason { get; } = reason; + + public override string ToString() => Target is null ? $"{Action}: {Reason}" : $"{Action} -> {Target}: {Reason}"; +} + +/// +/// Decides what a MOVING notification means for this connection, and where to go. +/// +/// +/// Separated from the acting so that it can be tested exhaustively without a server: the caller supplies the +/// endpoint, the address it is currently on, the announced window and a resolver, and gets back an action. +/// +/// The dispatch turns on the *form* of the endpoint, which is the thing that decides whether a handoff is even +/// possible - and the answer corrects an earlier assumption that MOVING should reuse the endpoint +/// retirement path: +/// +/// +/// +/// A hostname endpoint with no named successor - the case every observed MOVING has been - keeps its +/// : only the address behind the name moves. Retiring the endpoint would delete our +/// only way of reaching the deployment. So the action is to recycle the connections once DNS has moved, which +/// re-resolves them. +/// +/// +/// An address endpoint with a named successor is genuinely a different endpoint, so the topology has to be +/// re-read. Never observed: eleven routes have all carried an explicit null, including cases where the server +/// had already chosen the replacement node. Treat this branch as code that must exist rather than code that is +/// exercised. +/// +/// +/// An address endpoint with no successor has nothing to re-resolve and nowhere named to go. Doing nothing is +/// correct: the socket closes, the reconnect happens, and the relaxed window covers it. +/// +/// +/// +internal static class MaintenanceHandoff +{ + internal static async Task DecideAsync( + EndPoint endpoint, + EndPoint? successor, + IPAddress? currentAddress, + TimeSpan window, + TimeSpan pollInterval, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (successor is not null) + { + // Nothing is asserted about *which* endpoint: a successor may be an address we have never seen, so + // finding it is a topology question rather than a DNS one. + return new HandoffDecision(HandoffAction.Reconfigure, successor, "the server named a replacement endpoint"); + } + + if (endpoint is not DnsEndPoint dns) + { + return new HandoffDecision( + HandoffAction.None, + null, + $"{Format.ToString(endpoint)} is an address, not a name, and no replacement was named: nothing to re-resolve"); + } + + if (currentAddress is null) + { + // Without knowing where we are, "has it moved" is unanswerable - and guessing would mean recycling + // onto whatever DNS says right now, which for the first several seconds is the address being retired. + return new HandoffDecision(HandoffAction.None, null, "the address of the current connection is unknown"); + } + + var replacement = await AdvertisedAddressProbe.ProbeAsync( + dns, currentAddress, window, pollInterval, resolve, log, cancellationToken).ForAwait(); + + return replacement is null + ? new HandoffDecision( + HandoffAction.None, + null, + $"{dns.Host} still resolved only to {currentAddress} when the window expired") + : new HandoffDecision( + HandoffAction.Recycle, + replacement, + $"{dns.Host} now resolves to {replacement}"); + } + + /// + /// How long to wait before probing, so a fleet does not resolve in lockstep. + /// + /// + /// A *fraction* of the announced window rather than a fixed delay, which is the difference from the refresh + /// jitter elsewhere. Windows are not always generous - the shard notifications have been measured announcing + /// two seconds - so a flat one-second jitter could spend half of one, and on a short window the right amount + /// of jitter is almost none. Capped as well as scaled, because a long window does not justify a long wait + /// when DNS has been seen moving after four seconds. + /// + internal static TimeSpan GetJitter(TimeSpan window, Random random) + { + if (window <= TimeSpan.Zero) return TimeSpan.Zero; + + var tenth = window.TotalMilliseconds / 10; + var capped = Math.Min(tenth, 1000); + return TimeSpan.FromMilliseconds(random.Next(0, (int)Math.Max(capped, 1))); + } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs b/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs new file mode 100644 index 000000000..ae66e4ce2 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs @@ -0,0 +1,58 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// The kind of a server-native maintenance notification. +/// +/// +/// Two families share this enum: the Enterprise proxy notifications ( through +/// ) and the OSS cluster ones (, +/// ). Unrecognized types are dropped rather than surfaced, so no member here means +/// "something we didn't understand". +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceNotificationType +{ + /// + /// Not a maintenance notification; the default, for contexts where this is simply not applicable. + /// + None = 0, + + /// + /// This endpoint is being replaced; the notification names its successor, or names nothing at all if the + /// server has no address to offer. + /// + Moving, + + /// + /// A shard is migrating away from this node. Expect latency; expect a after it. + /// + Migrating, + + /// + /// A migration announced by has completed. + /// + Migrated, + + /// + /// This node is failing over. Expect latency; expect a after it. + /// + FailingOver, + + /// + /// A failover announced by has completed. + /// + FailedOver, + + /// + /// Slots are migrating (the OSS cluster family). + /// + SlotMigrating, + + /// + /// Slots announced by have migrated. + /// + SlotMigrated, +} diff --git a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs new file mode 100644 index 000000000..550d1e1de --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// A server-native maintenance notification, received as a RESP3 push frame on the connection that carries +/// commands. One class with a discriminator rather than a type per +/// notification: the payloads are near-identical, and it keeps the handling in one place. +/// +/// +/// Observation only at present: receiving one of these raises +/// and does nothing else. Acting on them - relaxing +/// timeouts, then moving off a doomed endpoint - is deliberately separate work, so a consumer can watch what +/// its servers are announcing before any behaviour depends on it. +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public sealed class PushMaintenanceEvent : ServerMaintenanceEvent +{ + internal PushMaintenanceEvent( + MaintenanceNotificationType notificationType, + long sequenceId, + EndPoint? endPoint, + TimeSpan? time, + EndPoint? newEndPoint, + string? payload, + string rawMessage, + IReadOnlyList? slotMigrations = null) + { + NotificationType = notificationType; + SequenceId = sequenceId; + EndPoint = endPoint; + Time = time; + NewEndPoint = newEndPoint; + Payload = payload; + RawMessage = rawMessage; + SlotMigrations = slotMigrations ?? []; + if (time is { } value && value > TimeSpan.Zero) + { + StartTimeUtc = ReceivedTimeUtc + value; + } + } + + /// + /// Which notification this is. + /// + public MaintenanceNotificationType NotificationType { get; } + + /// + /// The sequence number the server attached to this notification. + /// + /// + /// No specification defines these, but observation does: on Enterprise 8.6.2 they are monotonic per + /// database, start at zero on a fresh one, are shared *across* notification types (a + /// at 16 followed by its + /// at 17), and carry the same value on every node + /// that broadcasts a given event - so they identify the event rather than the connection that delivered + /// it. That makes them genuinely useful for spotting a replay. + /// + /// Still treat cross-deployment use as heuristic: this is one build of one product, and nothing obliges a + /// different implementation to behave the same way. + /// + /// + public long SequenceId { get; } + + /// + /// The server that sent the notification. + /// + /// + /// More precisely: whichever node told us first. Every node broadcasts a given event, so a three-proxy + /// deployment delivers one migration three times, on three connections, with the same + /// . All three are acted on internally - the timeout relaxation is per-server, so + /// each connection genuinely has to see it - but the event is raised once, for the first arrival, and the + /// rest are dropped. Do not read this as "the node being maintained": for the cluster notifications it is + /// usually a bystander reporting someone else's movements (see ). + /// + public EndPoint? EndPoint { get; } + + /// + /// How long the announced event is expected to take, or how much of it remains. + /// + /// + /// The meaning is per-notification: for it is the budget + /// for completing the move, and for the others it is the remaining duration of the announced disruption. + /// It can legitimately be zero or negative - a connection that arrives mid-window is told what is left of + /// it - which means "act now" rather than being an error. null where the notification carries no + /// time at all. + /// + public TimeSpan? Time { get; } + + /// + /// For , the endpoint this one is being replaced by. + /// + /// + /// null is a documented outcome, not just a parse failure: a server with no address to offer sends + /// an explicit null, and it also does so when it cannot honour the endpoint type that was requested. The + /// intended handling in that case is to reconnect using the endpoint already configured, rather than to + /// treat the notification as invalid. See for what actually arrived. + /// + public EndPoint? NewEndPoint { get; } + + /// + /// The final element of the notification, as received: the affected shard ids, the affected slots, or the + /// endpoint of a . + /// + /// + /// Deliberately opaque. Nothing a client is asked to *do* depends on which shards are involved, so this is + /// carried through for diagnostics rather than parsed into a model that the contract does not pin down. + /// + public string? Payload { get; } + + /// + /// For the cluster notifications, the slot movements described; empty for everything else. + /// + /// + /// A notification carries several of these, and the node that sent it is not necessarily the source of + /// any of them - every node reports the same movements. + /// + public IReadOnlyList SlotMigrations { get; } + + /// + public override string? ToString() => RawMessage; +} diff --git a/src/StackExchange.Redis/MaintenanceNotificationMode.cs b/src/StackExchange.Redis/MaintenanceNotificationMode.cs new file mode 100644 index 000000000..fc7c15623 --- /dev/null +++ b/src/StackExchange.Redis/MaintenanceNotificationMode.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis; + +/// +/// Whether to ask a server to send maintenance notifications - advance warning of migrations, failovers and +/// endpoint moves. +/// +/// +/// The three modes, and their names, are prescribed cross-client so that configuration and support tickets +/// line up between clients. Only Redis Enterprise and Redis Cloud emit these notifications; OSS Redis, Valkey +/// and Garnet do not recognize the opt-in at all, which is why exists and why the default +/// is rather than asking every server in existence. +/// +/// Note that means "required", not "on": it rejects any connection that cannot +/// deliver notifications. is the mode that turns the feature on where available and +/// stays out of the way otherwise, and is what most callers want. The names are the cross-client ones, and +/// this is the one place they are easy to misread. +/// +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceNotificationMode +{ + /// + /// Never ask; no notification is requested and none will arrive. + /// + Disabled = 0, + + /// + /// Requires them, and REJECTS THE CONNECTION if they are unavailable - including against any + /// server that does not support them, and on any RESP2 connection. Only for a deployment known to + /// support them; use to ask without that risk. + /// + /// + /// A server that refuses fails the connection, and so does a server that answers HELLO 3 as RESP2, + /// since nothing can be delivered on a RESP2 connection. So does a configuration that could never ask in + /// the first place - Protocol = Resp2, or HELLO unavailable - because requiring a + /// RESP3-only feature over RESP2 is a contradiction, and honouring half of it silently is the outcome + /// this mode exists to prevent. + /// + Enabled, + + /// + /// Ask, and carry on if the server refuses or the connection ends up RESP2 - the feature is then simply + /// off for that server, and the connection is never rejected over it. Safe against a mixture of servers, + /// or one whose support you don't know. + /// + Auto, +} diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 837651106..c6aceb60b 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -62,7 +62,10 @@ internal const CommandFlags NoFlushFlag = (CommandFlags)1024, // "server specific" (bit 18): tied to a specific endpoint, never retry elsewhere. Not (yet) a // public CommandFlags member - see the note on the hidden bit-18 value in CommandFlags.cs. - CommandServerSpecific = (CommandFlags)(1 << 18); + CommandServerSpecific = (CommandFlags)(1 << 18), + // "probe" (bit 19): health-check traffic. Deliberately *not* InternalCallFlag, which also decides + // queuing - see IsCallerFacing. + ProbeFlag = (CommandFlags)(1 << 19); protected RedisCommand command; @@ -92,7 +95,15 @@ internal const CommandFlags | CommandFlags.NoScriptCache | MaskRetryCategory // caller may override the retry category... | CommandServerSpecific // ...and the server-specific flag - | NoFlushFlag; // we'll allow this one even though not advertised + | NoFlushFlag // we'll allow this one even though not advertised + // ...and the probe flag, which *has* to survive this + // whitelist: health-check probes reach the pipeline + // through the public API (HealthCheckContext. + // ProbeFlags), so it arrives as caller-supplied flags + // or not at all. A caller passing it deliberately only + // opts their own command out of endpoint-idleness + // accounting, which is harmless. + | ProbeFlag; private IResultBox? resultBox; @@ -271,6 +282,32 @@ internal void WithHighIntegrity(uint value) public bool IsFireAndForget => (Flags & CommandFlags.FireAndForget) != 0; public bool IsInternalCall => (Flags & InternalCallFlag) != 0; + /// + /// Whether this is health-check traffic, issued by a + /// rather than by a caller. + /// + public bool IsProbe => (Flags & ProbeFlag) != 0; + + /// + /// Whether somebody outside the library is waiting on this message. + /// + /// + /// The question idleness actually wants to ask. Our own traffic - handshakes, heartbeats, + /// autoconfigure, health probes - is work we chose to do, so counting it makes a server look busy + /// precisely because we are looking at it, and an endpoint that can never be retired is the result + /// (see the endpoint-retirement work). + /// + /// A probe is a separate bit from on purpose. The internal-call flag also + /// decides *queuing*: an internal call bypasses the backlog + /// (PhysicalBridge.TryPushToBacklog) and is queued while disconnected regardless of the backlog + /// policy (QueueOrFailMessage). Bypassing the backlog would make a health check unable to + /// observe the one thing it most needs to - a bridge whose queue is not draining - and that signal is + /// what drives failover in a geo-redundant deployment. So probes are excluded from *accounting* without + /// being given internal-call *routing*. + /// + /// + public bool IsCallerFacing => (Flags & (InternalCallFlag | ProbeFlag)) == 0; + public IResultBox? ResultBox => resultBox; public abstract int ArgCount { get; } // note: over-estimate if necessary diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 9baa29d20..a739e7df8 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -390,6 +390,14 @@ internal void KeepAlive(bool forceRun = false) msg.SetSource(ResultProcessor.Tracer, null); break; case ConnectionType.Subscription: + // Normally the PING - observed against a 7.0 server, answered with the two-element array + // pong rather than +PONG (OnResponseFrame's IsArrayPong is what keeps that out of the + // out-of-band path so it still matches this message). The UNSUBSCRIBE fallback is not just + // for pre-3.0 servers: the condition also fails when PING is disabled or renamed in the + // CommandMap, or fronted by something that does not support it. + // + // Neither needs SetInternalCall here: the common path below flags whatever the switch + // produced, so any tracer added later is covered without remembering to. if (commandMap.IsAvailable(RedisCommand.PING) && features.PingOnSubscriber) { msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.PING); @@ -673,8 +681,12 @@ internal void OnHeartbeat(bool ifConnectedOnly) // This is an "always" check - we always want to evaluate a dead connection from a non-responsive sever regardless of the need to heartbeat above var totalTimeoutThisHeartbeat = asyncTimeoutThisHeartbeat + syncTimeoutThisHeartbeat; - bool deadConnectionOnAsync = asyncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds * 4); - bool deadConnectionOnSync = syncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.TimeoutMilliseconds * 4); + // note these track the *effective* timeout: this heuristic is derived from the + // command timeout, so if relaxation says "be patient for 60s" while this says + // "dead after 4x5s", we would tear down the connection relaxation was protecting. + // Socket-level failure detection is untouched and still notices a dead server. + bool deadConnectionOnAsync = asyncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (ServerEndPoint.GetEffectiveTimeoutMilliseconds(tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds ?? 0) * 4); + bool deadConnectionOnSync = syncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (ServerEndPoint.GetEffectiveTimeoutMilliseconds(tmp.BridgeCouldBeNull?.Multiplexer.TimeoutMilliseconds ?? 0) * 4); if (deadConnectionOnAsync || deadConnectionOnSync) { // If we've received *NOTHING* on the pipe in 4 timeouts worth of time and we're timing out commands, issue a connection failure so that we reconnect @@ -1019,10 +1031,29 @@ private void StartBacklogProcessor() /// Crawls from the head of the backlog queue, consuming anything that should have timed out /// and pruning it accordingly (these messages will get timeout exceptions). /// + /// + /// Whether this bridge owes a *caller* anything, ignoring our own internal traffic. + /// + internal bool HasCallerWork() + { + if (physical?.HasCallerMessagesAwaitingResponse() == true) return true; + + foreach (var message in _backlog) // snapshot enumeration; a concurrent queue is fine to walk + { + if (message.IsCallerFacing) return true; + } + return false; + } + private void CheckBacklogForTimeouts() { var now = Environment.TickCount; - var timeout = _singleWriter.TimeoutMilliseconds; + + // deliberately *not* _singleWriter.TimeoutMilliseconds, which is the write-lock acquisition + // timeout: that is about contention between writers, and relaxing it during maintenance is not + // something anybody asked for. This is the age at which a queued command is considered timed out, + // relaxed while the server has announced a disruption. + var timeout = ServerEndPoint.GetEffectiveTimeoutMilliseconds(Multiplexer.TimeoutMilliseconds); // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. @@ -1303,7 +1334,7 @@ public bool HasPendingCallerFacingItems() { foreach (var item in _backlog) // non-consuming, thread-safe, etc { - if (!item.IsInternalCall) return true; + if (item.IsCallerFacing) return true; } } return physical?.HasPendingCallerFacingItems() ?? false; @@ -1763,6 +1794,33 @@ internal void SimulateConnectionFailure(SimulatedFailureType failureType) physical?.SimulateConnectionFailure(failureType); } + /// + /// Drops the current connection so that a fresh one is established. + /// + /// + /// Deliberately just a dispose: that routes through RecordConnectionFailed to + /// , which reconnects immediately by itself, so there is no new lifecycle + /// here to get wrong. Used by the MOVING handoff, where the point is to choose *when* the + /// connection is replaced - after the replacement address is known - rather than waiting for the server + /// to close it and re-resolving to whatever DNS says at that moment, which has been measured as still + /// naming the node being retired. + /// + internal bool RecycleConnection(string reason) + { + var current = physical; + if (current is null) return false; + + Multiplexer.Trace($"recycling {ConnectionType} connection: {reason}", ToString()); + + // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises + // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and + // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first + // also performs the disconnect, so the Dispose below is cleanup. + current.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + current.Dispose(); + return true; + } + internal RedisCommand? GetActiveMessage() => Volatile.Read(ref _activeMessage)?.Command; } } diff --git a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs new file mode 100644 index 000000000..03e74f875 --- /dev/null +++ b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs @@ -0,0 +1,304 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using RESPite; +using RESPite.Messages; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +internal sealed partial class PhysicalConnection +{ + /// + /// Reads a maintenance notification and raises it as an event; observation only. + /// + /// + /// Parsed leniently on purpose. These frames are dispatched here *before* anything reads element 1 as a + /// channel name, because element 1 is a sequence number - so a malformed one must be swallowed here, and + /// never fall through to the command matcher, where it would take a reply belonging to something else. + /// The shapes are (per the contract, all elements after the type being scalars): + /// + /// MOVING seq time endpoint + /// MIGRATING seq time shards, FAILING_OVER seq time shards + /// MIGRATED seq shards, FAILED_OVER seq shards + /// SMIGRATING seq slots, SMIGRATED seq slots + /// + /// Integers may arrive as : or as a bulk string, trailing elements are explicitly allowed, and the + /// endpoint may be an explicit null. So the type decides *whether* a time is expected rather than the + /// content deciding it - a slot list of "123" must not be mistaken for a duration - but a notification + /// that omits or adds one is still accepted. + /// + private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, PushKind kind, ref RespReader reader) + { + _readStatus = ReadStatus.MaintenanceNotification; + + // at most three elements follow the type in any defined shape; anything beyond that is ignored + string? e1 = null, e2 = null, e3 = null; + List? migrations = null; + int count = 0; + while (reader.SafeTryMoveNext()) + { + count++; + if (!reader.IsScalar) + { + // SMIGRATED nests: [type, seq, [[source, target, slots], ...]]. Everything else is flat, and + // nesting there means a frame we do not understand - so the guard stays for those, since + // guessing at an unknown shape is how a parser starts inventing data + if (kind is PushKind.SlotMigrated or PushKind.SlotMigrating && migrations is null) + { + // note the reader has to be moved *past* the aggregate: enumerating the children does not + // advance it, so without this the loop walks back into the triplets we just read and + // mistakes them for further top-level elements + migrations = ReadSlotMigrations(ref reader); + continue; + } + + Trace($"{kind}: non-scalar element {count}"); + return OutOfBandResult.Handled; + } + + var element = reader.IsNull ? null : reader.ReadString(); + switch (count) + { + case 1: e1 = element; break; + case 2: e2 = element; break; + case 3: e3 = element; break; + } + } + + var type = ToNotificationType(kind); + + // A missing or unreadable sequence id is *not* fatal. The specs say every shape carries one, but + // go-redis - which runs against real servers - length-checks these frames at two elements and reads + // no sequence number at all for the shard notifications. So a client that drops a frame for want of a + // seq is stricter than one that demonstrably works. Without it we lose only dedup, which is our own + // invention anyway; the disruption being announced is the part that matters. + long? sequenceId = TryParseInt64(e1, out var parsedSequenceId) ? parsedSequenceId : null; + if (sequenceId is null) + { + OnMaintenanceNotificationDropped(type, $"no readable sequence id in a {count + 1}-element frame; continuing without dedup"); + } + + long? timeSeconds = null; + string? payload; + if (CarriesTime(type)) + { + if (TryParseInt64(e2, out var seconds)) + { + timeSeconds = seconds; + payload = e3; + } + else + { + // tolerated: a server that omits the time it was supposed to send + payload = e3 ?? e2; + } + } + else if (count >= 3 && TryParseInt64(e2, out var seconds)) + { + // tolerated the other way: a time on a notification the contract says has none + timeSeconds = seconds; + payload = e3; + } + else + { + payload = e3 ?? e2; + } + + var server = BridgeCouldBeNull?.ServerEndPoint; + EndPoint? newEndPoint = null; + if (type == MaintenanceNotificationType.Moving && !string.IsNullOrEmpty(payload)) + { + // "?" and an empty host are the contract's placeholders for "no address", and are no more + // dialable than the null form; they must never be taken to mean "the server that told us" + newEndPoint = ParseMigrationEndPoint(payload); + if (newEndPoint is null) + { + Trace($"{kind}: no usable endpoint in '{payload}'"); + } + } + + var time = timeSeconds is { } value ? TimeSpan.FromSeconds(value) : (TimeSpan?)null; + var raw = Describe(kind, sequenceId, timeSeconds, payload); + Trace($"maintenance notification: {raw}"); + OnDetailLog($"maintenance notification: {raw}"); + + // relax before reporting: the event handler is consumer code, and the window should already be open + // by the time anyone sees the notification that opened it + if (server is not null) + { + if (IsWindowOpening(type)) + { + var isNew = server.OnMaintenanceWindowOpened(type, sequenceId, time); + + // ...and MOVING alone means "this endpoint is going away", which is worth acting on rather than + // waiting to be disconnected. + // + // Only when the notification is *new*, and that is load-bearing rather than tidy. A server + // re-sends MOVING to a connection that opts in while the window is still open - measured, and + // the handoff replaces the connection, so acting on the repeat is a feedback loop: recycle, + // reconnect, get told again, recycle. It produced twelve recycles from one event on a real + // deployment before this guard. The per-server sequence dedup already knew it was a repeat; the + // handoff simply was not asking. + if (isNew && type == MaintenanceNotificationType.Moving) + { + server.OnMovingAnnounced(time, newEndPoint, this); + } + } + else if (IsWindowClosing(type)) + { + server.OnMaintenanceWindowClosed(type, sequenceId); + + // ...and if slots moved away from us, learn the new topology rather than waiting to be told + // by a -MOVED. Scoped and jittered inside OnSlotsMigratedAway; see its remarks for why this + // is the cluster family only + if (type == MaintenanceNotificationType.SlotMigrated && migrations is not null) + { + server.OnSlotsMigratedAway(migrations); + } + } + } + + // Per-server work above, one event below: relaxation is per-connection and every connection is told, + // but a consumer wants one callback per logical event rather than one per proxy that mentioned it + if (muxer.TryClaimMaintenanceEvent(type, sequenceId)) + { + var evt = new PushMaintenanceEvent(type, sequenceId ?? 0, server?.EndPoint, time, newEndPoint, payload, raw, migrations); + muxer.OnServerMaintenanceEvent(evt); + } + else + { + Trace($"{kind} seq {sequenceId} already reported by another node; not raising again"); + } + + return OutOfBandResult.Handled; + } + + /// + /// Reads the nested [[source, target, slots], ...] form of a cluster slot-migration notification. + /// + /// + /// A malformed triplet is skipped rather than abandoning the whole notification - the same choice go-redis + /// makes, and the right one: the other triplets are still actionable, and one bad entry should not lose a + /// migration we could have applied. The slot list is a flat comma-and-range string inside each triplet. + /// + private List ReadSlotMigrations(ref RespReader reader) + { + var results = new List(); + var outer = reader.AggregateChildren(); + while (outer.MoveNext()) + { + var triplet = outer.Value; + if (!triplet.IsAggregate) + { + Trace("slot migration: expected a triplet"); + continue; + } + + string? source = null, target = null, slots = null; + int index = 0; + var inner = triplet.AggregateChildren(); + while (inner.MoveNext()) + { + var child = inner.Value; + var text = child.IsScalar && !child.IsNull ? child.ReadString() : null; + switch (index++) + { + case 0: source = text; break; + case 1: target = text; break; + case 2: slots = text; break; + } + } + + if (index < 3) + { + Trace($"slot migration: {index}-element triplet, skipped"); + continue; + } + + var ranges = SlotRange.TryParseList(slots, out var parsed) ? parsed : []; + results.Add(new ClusterSlotMigration(ParseMigrationEndPoint(source), ParseMigrationEndPoint(target), ranges, slots)); + } + + // leave the caller's reader after the aggregate, so it can carry on with any trailing elements + outer.MovePast(out reader); + return results; + } + + /// + /// Parses one end of a migration, treating the placeholder forms as "not named" rather than as an error. + /// + private static EndPoint? ParseMigrationEndPoint(string? value) + => string.IsNullOrEmpty(value) || value == "?" || !Format.TryParseEndPoint(value, out var parsed) || IsPlaceholderEndPoint(parsed) + ? null + : parsed; + + private static bool IsPlaceholderEndPoint(EndPoint endpoint) => endpoint switch + { + DnsEndPoint dns => dns.Port == 0 || dns.Host is "" or "?", + IPEndPoint ip => ip.Port == 0, + _ => false, + }; + + private void OnMaintenanceNotificationDropped(MaintenanceNotificationType type, string reason) + { + // never fatal: a notification we cannot read is a diagnostic, not a protocol failure - the frame has + // been consumed either way, so the connection is not at risk + Trace($"dropped {type} notification: {reason}"); + OnDetailLog($"dropped {type} notification: {reason}"); + } + + private static bool TryParseInt64(string? value, out long result) + => long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out result); + + /// + /// Whether this notification announces a disruption starting (or still running). + /// + /// + /// is an opener with no closer: its window can only end + /// by the deadline the server gave us, or - later - by the handoff completing. + /// + private static bool IsWindowOpening(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Moving + or MaintenanceNotificationType.Migrating + or MaintenanceNotificationType.FailingOver + or MaintenanceNotificationType.SlotMigrating; + + /// + /// Whether this notification announces that a disruption has finished. + /// + private static bool IsWindowClosing(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.FailedOver + or MaintenanceNotificationType.SlotMigrated; + + /// + /// Whether the contract gives this notification a time element. + /// + private static bool CarriesTime(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Moving + or MaintenanceNotificationType.Migrating + or MaintenanceNotificationType.FailingOver; + + private static MaintenanceNotificationType ToNotificationType(PushKind kind) => kind switch + { + PushKind.Moving => MaintenanceNotificationType.Moving, + PushKind.Migrating => MaintenanceNotificationType.Migrating, + PushKind.Migrated => MaintenanceNotificationType.Migrated, + PushKind.FailingOver => MaintenanceNotificationType.FailingOver, + PushKind.FailedOver => MaintenanceNotificationType.FailedOver, + PushKind.SlotMigrating => MaintenanceNotificationType.SlotMigrating, + PushKind.SlotMigrated => MaintenanceNotificationType.SlotMigrated, + _ => MaintenanceNotificationType.None, + }; + + private static string Describe(PushKind kind, long? sequenceId, long? timeSeconds, string? payload) + { + var sb = new System.Text.StringBuilder(kind.ToString().ToUpperInvariant()); + sb.Append(" seq=").Append(sequenceId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "?"); + if (timeSeconds is { } seconds) sb.Append(" time=").Append(seconds).Append('s'); + if (payload is not null) sb.Append(' ').Append(payload); + return sb.ToString(); + } +} diff --git a/src/StackExchange.Redis/PhysicalConnection.Read.cs b/src/StackExchange.Redis/PhysicalConnection.Read.cs index e65941021..91efe09c9 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Read.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Read.cs @@ -508,11 +508,36 @@ internal enum PushKind PUnsubscribe, [AsciiHash("sunsubscribe")] SUnsubscribe, + + // the maintenance-notification family; these are *not* pub/sub - element 1 is a sequence number + // rather than a channel, so they must be dispatched before anything reads a channel name. The specs + // write them uppercase while the pub/sub kinds above are lowercase, hence the case-insensitive match + [AsciiHash("MOVING")] + Moving, + [AsciiHash("MIGRATING")] + Migrating, + [AsciiHash("MIGRATED")] + Migrated, + [AsciiHash("FAILING_OVER")] + FailingOver, + [AsciiHash("FAILED_OVER")] + FailedOver, + [AsciiHash("SMIGRATING")] + SlotMigrating, + [AsciiHash("SMIGRATED")] + SlotMigrated, } internal static partial class PushKindMetadata { - [AsciiHash] + /// + /// Identifies a push frame from its first element. + /// + /// + /// Case-insensitive: the pub/sub kinds are lowercase on the wire and the maintenance kinds are + /// uppercase, and no specification anywhere is careful about it - so don't bake in an assumption. + /// + [AsciiHash(CaseSensitive = false)] internal static partial bool TryParse(ReadOnlySpan value, out PushKind result); } @@ -563,6 +588,12 @@ private OutOfBandResult OnOutOfBand(ReadOnlySpan payload, ref IMemoryOwner if (!reader.TryParseScalar(&PushKindMetadata.TryParse, out kind)) kind = PushKind.None; } + if (kind is >= PushKind.Moving and <= PushKind.SlotMigrated) + { + // not pub/sub: dispatch before anything tries to read element 1 as a channel name + return OnMaintenanceNotification(muxer, kind, ref reader); + } + RedisChannel.RedisChannelOptions channelOptions = kind switch { PushKind.PMessage or PushKind.PSubscribe or PushKind.PUnsubscribe => RedisChannel.RedisChannelOptions.Pattern, diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 1c4222584..c16cfa40a 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -811,6 +811,31 @@ internal void GetStormLog(StringBuilder sb) } } + /// + /// Whether any message awaiting a response was issued by a *caller* rather than by us. + /// + /// + /// The distinction matters wherever we ask "is anyone using this server": our own handshake, + /// autoconfigure and keep-alive traffic is not use, and treating it as such makes a server we are + /// probing look busy *because* we are probing it. + /// + /// A predicate rather than a count, because every caller only asks whether the answer is zero - and + /// this way the common case (a caller's message at the head of the queue) costs one iteration instead + /// of walking a queue that a stalled server can leave thousands of entries long. + /// + /// + internal bool HasCallerMessagesAwaitingResponse() + { + lock (_writtenAwaitingResponse) + { + foreach (var message in _writtenAwaitingResponse) + { + if (message.IsCallerFacing) return true; + } + } + return false; + } + /// /// Runs on every heartbeat for a bridge, timing out any commands that are overdue and returning an integer of how many we timed out. /// @@ -829,7 +854,9 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou { var server = bridge.ServerEndPoint; var multiplexer = bridge.Multiplexer; - var timeout = multiplexer.AsyncTimeoutMilliseconds; + + // relaxed while this server has announced a disruption; a floor, never a reduction + var timeout = server.GetEffectiveTimeoutMilliseconds(multiplexer.AsyncTimeoutMilliseconds); foreach (var msg in _writtenAwaitingResponse) { // We only handle async timeouts here, synchronous timeouts are handled upstream. @@ -1272,6 +1299,7 @@ internal enum ReadStatus ResetArena, ProcessBufferComplete, PubSubUnsubscribe, + MaintenanceNotification, NA = -1, } @@ -1287,7 +1315,7 @@ internal bool HasPendingCallerFacingItems() { foreach (var item in _writtenAwaitingResponse) { - if (!item.IsInternalCall) return true; + if (item.IsCallerFacing) return true; } } return false; diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 419c74dc2..4b87978f5 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -111,3 +111,68 @@ static StackExchange.Redis.BitFieldOperation.IncrementBy(StackExchange.Redis.Bit static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.Set(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset, long value, StackExchange.Redis.BitFieldOverflow overflow = StackExchange.Redis.BitFieldOverflow.Wrap) -> StackExchange.Redis.BitFieldOperation +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.set -> void +[SER010]StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Auto = 2 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Disabled = 0 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Enabled = 1 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Maintenance.PushMaintenanceEvent.ToString() -> string? +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailedOver = 5 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailingOver = 4 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrated = 3 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrating = 2 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Moving = 1 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.None = 0 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrated = 7 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrating = 6 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.EndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NewEndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NotificationType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Payload.get -> string? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SequenceId.get -> long +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Time.get -> System.TimeSpan? +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.AbortOnConnectFail.get -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.IsMatch(System.Net.EndPoint! endpoint) -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +[SER010]override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +StackExchange.Redis.Configuration.RedisCloudOptionsProvider +StackExchange.Redis.Configuration.RedisCloudOptionsProvider.RedisCloudOptionsProvider() -> void +virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.Name.get -> string? +override StackExchange.Redis.Configuration.AzureOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +[SER010]override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.RedisEnterpriseOptionsProvider() -> void +override StackExchange.Redis.Configuration.DefaultOptionsProvider.ToString() -> string! +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.set -> void +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan? +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedWindowMax.get -> System.TimeSpan? +[SER010]StackExchange.Redis.Availability.FaultContext.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.RedisConnectionException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.RedisTimeoutException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]override StackExchange.Redis.Maintenance.ClusterSlotMigration.ToString() -> string! +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.ClusterSlotMigration() -> void +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.RawSlots.get -> string? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Slots.get -> System.Collections.Generic.IReadOnlyList! +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Source.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Target.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SlotMigrations.get -> System.Collections.Generic.IReadOnlyList! +[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeFlags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.ConnectionFailureType.MaintenanceHandoff = 12 -> StackExchange.Redis.ConnectionFailureType diff --git a/src/StackExchange.Redis/RedisLiterals.cs b/src/StackExchange.Redis/RedisLiterals.cs index 12b85ea2f..d66cb841f 100644 --- a/src/StackExchange.Redis/RedisLiterals.cs +++ b/src/StackExchange.Redis/RedisLiterals.cs @@ -72,6 +72,7 @@ public static readonly RedisValue LIMIT = RedisValue.FromRaw("LIMIT"u8), LIST = RedisValue.FromRaw("LIST"u8), LT = RedisValue.FromRaw("LT"u8), + MAINT_NOTIFICATIONS = RedisValue.FromRaw("MAINT_NOTIFICATIONS"u8), MATCH = RedisValue.FromRaw("MATCH"u8), MALLOC_STATS = RedisValue.FromRaw("MALLOC-STATS"u8), MAX = RedisValue.FromRaw("MAX"u8), @@ -97,6 +98,8 @@ public static readonly RedisValue PATTERN = RedisValue.FromRaw("PATTERN"u8), PAUSE = RedisValue.FromRaw("PAUSE"u8), PERSIST = RedisValue.FromRaw("PERSIST"u8), + OFF = RedisValue.FromRaw("OFF"u8), + ON = RedisValue.FromRaw("ON"u8), PING = RedisValue.FromRaw("PING"u8), PREPARE = RedisValue.FromRaw("PREPARE"u8), PURGE = RedisValue.FromRaw("PURGE"u8), diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 2175294e8..f8eb06a0e 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -25,6 +25,7 @@ public static readonly ResultProcessor TrackSubscriptions = new TrackSubscriptionsProcessor(null), Tracer = new TracerProcessor(false), EstablishConnection = new TracerProcessor(true), + MaintenanceNotifications = new MaintenanceNotificationsProcessor(), BackgroundSaveStarted = new ExpectBasicStringProcessor(Literals.background_saving_started.Hash, startsWith: true), BackgroundSaveAOFStarted = new ExpectBasicStringProcessor(Literals.background_aof_rewriting_started.Hash, startsWith: true); @@ -3191,6 +3192,44 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } + /// + /// Handles the reply to the maintenance-notification opt-in, which we send speculatively: a server that + /// doesn't know the subcommand replies with an error, and that is an expected outcome rather than a + /// fault. So the error is absorbed here rather than going through the common error path, which would + /// raise an to the consumer for something we asked for + /// on their behalf. + /// + private sealed class MaintenanceNotificationsProcessor : ResultProcessor + { + public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) + { + reader.MovePastBof(); + var server = connection.BridgeCouldBeNull?.ServerEndPoint; + if (reader.IsError) + { + server?.OnMaintenanceNotificationsRefused(connection, reader.ReadString() ?? "declined"); + SetResult(message, false); + return true; + } + + if (reader.IsScalar && Literals.OK.Hash.IsCS(reader.TryGetSpan(out var span) ? span : reader.Buffer(stackalloc byte[16]))) + { + server?.OnMaintenanceNotificationsAccepted(); + SetResult(message, true); + return true; + } + + // anything else: treat as "not available" rather than a protocol fault; being liberal in what + // we accept matters more here than pinning an unverifiable reply shape + server?.OnMaintenanceNotificationsRefused(connection, $"unexpected reply: {reader.GetOverview()}"); + SetResult(message, false); + return true; + } + + protected override bool SetResultCore(PhysicalConnection connection, Message message, ref RespReader reader) + => throw new NotSupportedException(); // SetResult is fully overridden + } + private sealed class TracerProcessor(bool establishConnection) : ResultProcessor { public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs new file mode 100644 index 000000000..28f00e1b0 --- /dev/null +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -0,0 +1,591 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +internal sealed partial class ServerEndPoint +{ + private volatile bool _maintenanceNotificationsActive, _maintenanceNotificationsRequested; + private volatile string? _maintenanceNotificationsRefusal; + + /// + /// Whether this server has accepted our request for maintenance notifications on the current connection. + /// + /// + /// Per-connection state, so this is cleared at the start of every handshake and re-established from the + /// reply; the scope is deliberately the server rather than the multiplexer, so one lagging node doesn't + /// disable the feature for the whole deployment. + /// + internal bool MaintenanceNotificationsActive => _maintenanceNotificationsActive; + + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + private MaintenanceNotificationMode MaintenanceMode + => Multiplexer.RawConfig.MaintenanceNotifications; + + /// + /// Whether to ask this server for maintenance notifications during handshake. + /// + /// + /// The feature is RESP3-only: the notifications are out-of-band push frames on the connection that + /// carries commands, and a RESP2 connection can have the request accepted and then silently receive + /// nothing - so we don't ask unless we asked for RESP3. (Note that not asking is not the same as being + /// satisfied: under the reconcile below fails the + /// connection for exactly this case.) We can't know what was *negotiated* at write time + /// (the handshake is pipelined, and HELLO hasn't been answered yet), so this tests what we asked + /// for and settles it once the reply has been processed. + /// + private bool ShouldRequestMaintenanceNotifications(bool isInteractive, bool negotiateResp3) + => isInteractive + && negotiateResp3 + && MaintenanceMode != MaintenanceNotificationMode.Disabled + && Multiplexer.CommandMap.IsAvailable(RedisCommand.CLIENT); + + internal void OnMaintenanceNotificationsAccepted() => _maintenanceNotificationsActive = true; + + /// + /// The server declined our request. Recorded rather than acted on: whether that matters is a question for + /// , which sees the negotiated protocol too. + /// + internal void OnMaintenanceNotificationsRefused(PhysicalConnection connection, string reason) + { + _maintenanceNotificationsActive = false; + _maintenanceNotificationsRefusal = reason; + connection.OnDetailLog($"maintenance notifications refused: {reason}"); + } + + /// + /// Settles the feature for this connection now that the handshake is complete and the protocol is known. + /// + /// + /// The opt-in reply precedes the tracer on the same pipelined connection, so by the time this runs every + /// fact is in: whether we asked, what the server said, and what protocol we ended up on. + /// + private void ReconcileMaintenanceNotifications(PhysicalConnection connection) + { + bool resp3 = connection.Protocol is >= RedisProtocol.Resp3; + if (!resp3) + { + // whatever the server said about the opt-in, nothing can arrive on a RESP2 connection + _maintenanceNotificationsActive = false; + } + + if (_maintenanceNotificationsActive || MaintenanceMode != MaintenanceNotificationMode.Enabled) + { + return; + } + + // Enabled means required: no notifications, no connection. That includes a configuration that never + // got as far as asking - requiring a RESP3-only feature over RESP2 is a contradiction, and failing it + // is more useful than honouring half of it. Note the cross-client spec only calls for failing when + // the *server* errors; extending that to the RESP2 cases is ours, and deliberate + var reason = !resp3 + ? (_maintenanceNotificationsRequested ? "the connection negotiated RESP2" : "RESP3 was not requested") + : _maintenanceNotificationsRefusal ?? "the server did not accept the request"; + + connection.RecordConnectionFailed( + ConnectionFailureType.ProtocolFailure, + new RedisConnectionException(ConnectionFailureType.ProtocolFailure, CommandFlags.None, $"Maintenance notifications are enabled, but unavailable: {reason}", innerException: null)); + } + + // The relaxed-timeout window, expressed as a single deadline in Environment.TickCount terms so that it + // can be read without a lock from the heartbeat sweeps. Zero means "no window"; a deadline that computes + // to zero is nudged by a tick rather than complicating the sentinel. Comparisons are unchecked + // subtraction, as elsewhere in this type, so the ~49-day wrap is a non-event. + private int _relaxedDeadlineTicks; + + // the notification that last touched the window, reported on faults that happen inside it + private int _relaxedType; + + // Dedup state, per notification type. No specification defines the sequence ids, but they were observed on + // Enterprise 8.6.2 to be monotonic per database and shared across types, and to carry the same value on + // every node broadcasting a given event - so they identify the event, which is exactly what dedup needs. + // Keyed per type anyway: within a type the ids are still monotonic, and a per-type key cannot mistake one + // node's earlier event for a replay of another's later one. Allocated on first notification, so a server + // that never sees one pays nothing. + private long[]? _lastSequenceIds; + private bool[]? _haveSequenceIds; // zero is a real sequence number, so "unset" needs its own bit + private readonly object _maintenanceSync = new(); + + /// + /// Whether timeouts are currently relaxed for this server. + /// + internal bool IsMaintenanceRelaxed => GetRelaxedRemaining() > 0; + + /// + /// The notification in force for this server, or if no + /// window is open; reported on faults so that a timeout during a migration says so. + /// + internal MaintenanceNotificationType ActiveMaintenanceType + => GetRelaxedRemaining() > 0 ? (MaintenanceNotificationType)Volatile.Read(ref _relaxedType) : MaintenanceNotificationType.None; + + /// + /// The effective timeout for a command against this server: the configured value, or the relaxed value if + /// that is larger and a window is open. Relaxation is a floor and can never shorten a timeout. + /// + /// + /// Read from the timeout sweeps rather than stamped onto each message: both sweeps rely on head-of-line + /// ordering and stop at the first message that has not timed out, which per-message timeouts would + /// invalidate. The consequence is that relaxation applies to whatever is outstanding when a window opens, + /// and stops applying when it closes - which is one of the reasons the post-event tail exists. + /// + internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) + { + if (GetRelaxedRemaining() <= 0) return configuredMilliseconds; + + var relaxed = (int)Multiplexer.RawConfig.MaintenanceRelaxedTimeout.TotalMilliseconds; + return relaxed > configuredMilliseconds ? relaxed : configuredMilliseconds; + } + + /// + /// Milliseconds of relaxation left, or zero if none; clears an expired window as a side-effect. + /// + private int GetRelaxedRemaining() + { + var deadline = Volatile.Read(ref _relaxedDeadlineTicks); + if (deadline == 0) return 0; + + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining > 0) return remaining; + + // expired; clear it, but only if nobody has moved it on in the meantime + Interlocked.CompareExchange(ref _relaxedDeadlineTicks, 0, deadline); + return 0; + } + + /// + /// An announced disruption has started (or is still running): open or extend the relaxed window. + /// + /// + /// Whether this notification was new. A replay extends nothing and, importantly, must not be *acted* on - + /// see the handoff, where re-acting on a repeat is a feedback loop rather than merely wasted work. + /// + internal bool OnMaintenanceWindowOpened(MaintenanceNotificationType type, long? sequenceId, TimeSpan? time) + { + if (!TryClaimSequenceId(type, sequenceId)) return false; + + var config = Multiplexer.RawConfig; + var floor = config.MaintenanceRelaxedTimeout; + var cap = config.MaintenanceRelaxedWindowMax; + + // no time, or a time at or below zero, means "act now" rather than "ignore this": a connection that + // arrives mid-window is told what is left of it, and that can legitimately be negative + var duration = time is { } value && value > TimeSpan.Zero ? value : floor; + if (duration < floor) duration = floor; + if (duration > cap) duration = cap; + + Volatile.Write(ref _relaxedType, (int)type); + ExtendRelaxedWindow(duration, $"{type} for {duration.TotalSeconds}s"); + return true; + } + + /// + /// An announced disruption has finished: replace the remaining window with the post-event tail. + /// + /// + /// Replace rather than extend-or-shorten. The server has told us the operation completed, so whatever it + /// previously said about duration is stale - but the tail still applies, because completion is when every + /// other client that received the same notification re-engages. + /// + internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? sequenceId) + { + if (!TryClaimSequenceId(type, sequenceId)) return; + + Volatile.Write(ref _relaxedType, (int)type); + var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; + if (tail <= TimeSpan.Zero) + { + Volatile.Write(ref _relaxedDeadlineTicks, 0); + Multiplexer.Trace($"{type}: relaxation ended", ToString()); + return; + } + + var deadline = NudgeFromZero(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); + Volatile.Write(ref _relaxedDeadlineTicks, deadline); + Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); + } + + private int _handoffInFlight, _handoffRecycles; + private volatile string? _lastHandoffOutcome; + + /// + /// What the last MOVING handoff decided, and why. + /// + /// + /// Recorded rather than only traced because Multiplexer.Trace is [Conditional("VERBOSE")] - it + /// compiles away in any normal build, so a handoff that misbehaves in production would leave no trace at + /// all. This is the minimum that survives: what was decided, readable afterwards. + /// + internal string? LastHandoffOutcome => _lastHandoffOutcome; + + /// How many times a handoff has replaced this server's connections. + internal int HandoffRecycles => Volatile.Read(ref _handoffRecycles); + + /// + /// Acts on a MOVING: find where to go, then get off this connection before we are pushed. + /// + /// + /// The value here is entirely in the *timing*. Without it we already survive a MOVING - the socket + /// closes and we reconnect - but the reconnect happens when the server decides, and re-resolves to whatever + /// DNS says at that moment, which has been measured as still naming the node being retired. Measured on a + /// real deployment: MOVING arrives about six seconds in, DNS moves somewhere between four and + /// nineteen seconds later, and the socket closes seventeen to nineteen seconds after the notification. So + /// the window exists to be *used*, and using it means waiting for DNS to move and then choosing the moment. + /// + /// Fire-and-forget by design: this runs while the connection it concerns is still serving commands, and the + /// notification is delivered on the read loop, which must not wait for a DNS poll. + /// + /// + internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalConnection connection) + { + // One at a time per server. A rolling operation delivers one MOVING per connection, so a second one + // arriving while a handoff is in flight is a repeat or a much later event; either way, starting a + // second poll against the same endpoint achieves nothing. + if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) + { + Multiplexer.Trace("MOVING: a handoff is already in flight", ToString()); + return; + } + + var budget = window is { } value && value > TimeSpan.Zero + ? value + : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor + var current = (connection.VolatileSocket?.RemoteEndPoint as IPEndPoint)?.Address; + + _ = HandoffAsync(budget, successor, current); + } + + private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? currentAddress) + { + try + { + // Spread the fleet, but only by a fraction of the window - see MaintenanceHandoff.GetJitter for why + // a flat delay is wrong here. + var jitter = MaintenanceHandoff.GetJitter(window, RandomFor(this)); + if (jitter > TimeSpan.Zero) await Task.Delay(jitter).ForAwait(); + + var remaining = window - jitter; + var decision = await MaintenanceHandoff.DecideAsync( + EndPoint, + successor, + currentAddress, + remaining, + pollInterval: TimeSpan.FromSeconds(1), // records carry a 5s TTL, so this is several looks per record + resolve: Multiplexer.AddressResolver, + log: message => Multiplexer.Trace(message, ToString())).ForAwait(); + + Multiplexer.Trace($"MOVING: {decision}", ToString()); + _lastHandoffOutcome = decision.ToString(); + switch (decision.Action) + { + case HandoffAction.Recycle: + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + break; + case HandoffAction.Reconfigure: + // A named successor is a different endpoint, so this is a topology question. Note no + // observed deployment has ever named one, so this path has never run outside a test. + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, "moving names a successor"); + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + break; + default: + // Nothing to do is a legitimate outcome, not a failure: the server closes the socket, the + // reconnect re-resolves, and the relaxed window covers the gap. + break; + } + } + catch (Exception ex) + { + Multiplexer.OnInternalError(ex, EndPoint); + } + finally + { + Volatile.Write(ref _handoffInFlight, 0); + } + } + + /// + /// Lets in-flight caller work finish, then replaces the connections. + /// + /// + /// Draining first because the socket is still working: anything already written may still be answered, and + /// dropping it would fail commands that were about to succeed. Bounded by what is left of the window, + /// because the server closes the socket at the end of it regardless - so anything not drained by then was + /// going to fail either way, and draining strictly dominates. + /// + /// Both bridges, not just the one that was told. The measured blast radius is the *node*: four connections + /// to one proxy, differing only in handshake, all closed simultaneously, and only the ones that had opted in + /// were warned. + /// + /// + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) + { + var watch = ValueStopwatch.StartNew(); + while (HasCallerWork() && watch.ElapsedMilliseconds < budget.TotalMilliseconds) + { + await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); + } + + var drained = !HasCallerWork(); + var recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); + if (recycled) Interlocked.Increment(ref _handoffRecycles); + Multiplexer.Trace( + $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" + + (drained ? " (drained)" : " (still busy; the window ran out)"), + ToString()); + } + + [ThreadStatic] + private static Random? _random; + + /// + /// A per-thread , seeded so that two processes handing off at once do not pick the same + /// jitter. + /// + private static Random RandomFor(ServerEndPoint server) => + _random ??= new Random(Environment.TickCount ^ server.GetHashCode()); + + private void ExtendRelaxedWindow(TimeSpan duration, string cause) + { + var candidate = NudgeFromZero(unchecked(Environment.TickCount + (int)duration.TotalMilliseconds)); + while (true) + { + var current = Volatile.Read(ref _relaxedDeadlineTicks); + + // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer + // MIGRATING window must not cut it short + if (current != 0 && unchecked(candidate - current) <= 0) return; + if (Interlocked.CompareExchange(ref _relaxedDeadlineTicks, candidate, current) == current) + { + Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); + return; + } + } + } + + /// + /// Zero is the "no window" sentinel, so a deadline that lands on it moves by a tick. + /// + private static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; + + /// + /// Whether this notification is new, per the conservative dedup described on . + /// + private bool TryClaimSequenceId(MaintenanceNotificationType type, long? sequenceId) + { + // no id, no dedup - but the notification still counts. Dropping an announced disruption because we + // could not read a field whose meaning nobody has defined would be the wrong way round + if (sequenceId is not { } id) return true; + + var index = (int)type; + lock (_maintenanceSync) + { + var ids = _lastSequenceIds ??= new long[MaintenanceNotificationTypeCount]; + var have = _haveSequenceIds ??= new bool[MaintenanceNotificationTypeCount]; + if ((uint)index >= (uint)ids.Length) return true; // unknown type: don't dedup what we can't index + + // The "have we seen one" bit is separate because zero is a real sequence number - observed on + // Enterprise 8.6.2 as the first event of a chain (`>4 $6 MOVING :0 :15 _`). Treating a stored zero + // as "unset", which an earlier cut did, quietly disabled dedup for whichever notification happened + // to open the chain. + // + // note "<=", not "<": a replay carries the id we already acted on + if (have[index] && id <= ids[index]) + { + Multiplexer.Trace($"{type}: ignoring replayed sequence id {id}", ToString()); + return false; + } + + ids[index] = id; + have[index] = true; + return true; + } + } + + private const int MaintenanceNotificationTypeCount = 8; // None + the seven notification types + + /// + /// How long to smear a notification-triggered topology refresh over. + /// + /// + /// Not configurable, deliberately. The relaxed-window durations are options because their right value is + /// deployment-specific and we invented them; this is a fixed small smear that exists only so that a fleet + /// which was all told the same thing at the same instant does not all query the same node at the same + /// instant. Nobody needs to tune it, and every option is public surface to keep. Promote it if that turns + /// out to be wrong. + /// + /// Note already declines while one refresh is in + /// flight, so the *local* storm is handled; this is purely about the fleet. + /// + /// + private static readonly int MaintenanceRefreshJitterMilliseconds = 1000; + + // 1 while a jittered refresh is scheduled and has not yet started + private int _refreshPending; + + /// + /// Whether this server is one of the sources in a slot-migration delta - i.e. whether the movement being + /// described is movement away from us. + /// + /// + /// Every node in the cluster reports the same movements, so the sender is not implicitly a source and most + /// notifications describe somebody else. Acting only on our own is what stops a fleet re-reading topology + /// because one shard moved somewhere unrelated - go-redis makes the same choice. + /// + /// Resolution goes through the identity map rather than comparing endpoints directly: a node answers to + /// its address and to its announced hostname, and the delta may name either. + /// + /// + private bool IsSourceOf(IReadOnlyList migrations) + { + foreach (var migration in migrations) + { + if (migration.Source is { } source && ReferenceEquals(Multiplexer.TryResolveServerEndPoint(source), this)) + { + return true; + } + } + + return false; + } + + /// + /// Refreshes the topology after slots have moved away from this server, once the fleet has had a moment to + /// spread out. + /// + /// + /// Deliberately only for the cluster family: MIGRATED and FAILED_OVER arrive in proxied + /// deployments where the client addresses a single endpoint, so there is no topology for a refresh to + /// learn - they get relaxation and nothing else. Endpoints left serving nothing are handled by the + /// existing absence-based pruning, which a refresh feeds; there is deliberately no second, faster + /// retirement path here. + /// + internal void OnSlotsMigratedAway(IReadOnlyList migrations) + { + if (migrations.Count == 0 || !IsSourceOf(migrations)) return; + + // Coalesce *before* the jitter, not after. ReconfigureIfNeeded declines only while a refresh is + // actually in flight, and the jitter spreads a burst of notifications out far enough that each one + // completes before the next begins - so relying on that alone turns ten notifications into ten + // topology passes. Found by the test that counts them. + if (Interlocked.CompareExchange(ref _refreshPending, 1, 0) != 0) + { + Multiplexer.Trace("topology refresh already pending; folding this notification into it", ToString()); + return; + } + + var cause = $"slots migrated from {Format.ToString(EndPoint)}"; + Multiplexer.Trace($"{cause}; refreshing topology after jitter", ToString()); + + // fire-and-forget: we are on the read loop, and the refresh must not block it + _ = RefreshAfterJitterAsync(cause, migrations); + } + + private async Task RefreshAfterJitterAsync(string cause, IReadOnlyList migrations) + { + try + { + await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(MaintenanceRefreshJitterMilliseconds)).ForAwait(); + + // deliberately *after* the delay - see the remarks on ResubscribeStrandedShardedChannels + ResubscribeStrandedShardedChannels(migrations); + + // released before the refresh runs, not after: anything that arrives from here on describes a + // state this pass may not have seen, and deserves its own pass + Volatile.Write(ref _refreshPending, 0); + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, cause); + } + catch (Exception ex) + { + // a refresh we failed to start is a missed optimization, not a fault: the next -MOVED still works + Volatile.Write(ref _refreshPending, 0); + Multiplexer.Trace($"topology refresh after {cause} failed: {ex.Message}", ToString()); + } + } + + /// + /// Re-establishes sharded subscriptions that the slot movement stranded and nothing else recovered. + /// + /// + /// Mostly belt-and-braces: a server that migrates a slot also sends an unsolicited SUNSUBSCRIBE, + /// and already resubscribes on that. This adds two things. + /// It is *pre-emptive* where SMIGRATED arrives first, and it *covers* the case where the + /// unsolicited unsubscribe never arrives or is lost - in which case the only other signal is a message + /// that silently stops being delivered, which nothing detects. + /// + /// It also knows which slots moved, so only the affected channels are touched rather than everything + /// subscribed on this server. + /// + /// + /// Why this runs after the jitter, and only for a subscription connected nowhere. An earlier cut ran + /// it immediately, on the reasoning that a silently-unsubscribed subscriber is a correctness problem while a + /// stale slot map is only a round trip. Measured against a fake that emits the realistic sequence, that + /// produced an extra resubscribe per channel - because the unsolicited unsubscribe had already started one, + /// and mid-flight the subscription still looked like ours. Being a genuine *fallback* means acting only + /// once the other path has had its chance, which costs a stranded subscription up to the jitter in + /// recovery time and costs nothing when it was not needed. + /// + /// + /// Measured against the fake's realistic sequence: four (re)subscribes with notifications off, five with + /// them on. The extra one is this fallback acting on a subscription the unsolicited-unsubscribe path left + /// attached to nothing - i.e. it is the feature working, not duplicate work. One extra attempt per + /// stranded channel, not one per notification. + /// + /// + /// Note it resubscribes via this server, not the migration target, reusing + /// unchanged: the outgoing node is the one we know has + /// the new route, and sending there follows the redirect. The target is named in the notification and + /// could be dialled directly, but it may be a node we have never seen, or named in a form we cannot dial, + /// and the redirect path is the one already proven by the SUNSUBSCRIBE case. + /// + /// + private void ResubscribeStrandedShardedChannels(IReadOnlyList migrations) + { + var subscriptions = Multiplexer.GetSubscriptions(); + if (subscriptions.IsEmpty) return; + + var strategy = Multiplexer.ServerSelectionStrategy; + foreach (var pair in subscriptions) + { + var channel = pair.Key; + + // ordinary pub/sub is not slot-bound, so a slot moving says nothing about it + if (!channel.IsSharded) continue; + + var slot = strategy.HashSlot(channel); + if (slot == ServerSelectionStrategy.NoSlot || !IsInMigratedRange(migrations, slot)) continue; + + // Skip only if it is attached somewhere *else* - that means the other path already moved it. Still + // attached to us is the pre-emptive case (the notification beat the unsubscribe, and the + // subscription is now stale); attached nowhere is the stranded case. Both need acting on, and + // distinguishing them from "already handled" is the whole job of this check. + var subscription = pair.Value; + var current = subscription.GetAnyCurrentServer(); + if (current is not null && !ReferenceEquals(current, this)) + { + Multiplexer.Trace($"slot {slot} moved, and {channel} has already moved with it; leaving it alone", ToString()); + continue; + } + + Multiplexer.Trace($"slot {slot} moved; resubscribing {channel}", ToString()); + Multiplexer.DefaultSubscriber.ResubscribeToServer(subscription, channel, this, cause: "smigrated"); + } + } + + private static bool IsInMigratedRange(IReadOnlyList migrations, int slot) + { + foreach (var migration in migrations) + { + foreach (var range in migration.Slots) + { + if (slot >= range.From && slot <= range.To) return true; + } + } + + return false; + } +} diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 17d37a5c0..49c408e06 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -331,7 +331,22 @@ internal bool IsIdle() => !Multiplexer.ServerSelectionStrategy.OwnsAnySlot(this) && (subscription?.SubscriptionCount ?? 0) == 0 && (interactive?.SubscriptionCount ?? 0) == 0 - && GetOutstandingCount() == 0; + && !HasCallerWork(); + + /// + /// Whether a *caller* is waiting on anything here, which is the only kind of work that should stop us + /// retiring a server. + /// + /// + /// Deliberately not , which counts everything. A node the topology has + /// stopped listing still receives our autoconfigure probes on every pass, and nothing answers them, so + /// they accumulate in its backlog: measured at ~170 per pass, growing without bound. Counting those + /// made the node look busy *because* we were looking for it, so it could never be retired - the + /// precondition defeated itself in exactly the case pruning exists for. Keep-alive traffic has the same + /// property, and is excluded by the same test, since both set the internal-call flag. + /// + internal bool HasCallerWork() + => interactive?.HasCallerWork() == true || subscription?.HasCallerWork() == true; /// /// Work this server still owes an answer on: written-and-awaiting-response, plus anything queued in @@ -360,18 +375,23 @@ internal async Task RetireAsync(string reason, TimeSpan drainTimeout, ILogger? l SetUnselectable(UnselectableFlags.Retiring); log?.LogInformationRetiringServer(new(EndPoint), reason); + // Drain what a *caller* is waiting for, not everything outstanding. Our own probes to a node that + // has gone away will never be answered, so draining on the total means always waiting out the full + // timeout before letting go - measured: a departed node accumulates our autoconfigure traffic + // indefinitely (500+ and climbing), so the drain never once completed early. Callers are who the + // drain exists for; nobody is waiting on our keep-alives. // Stopwatch rather than TickCount64: the latter does not exist on the down-level targets var watch = ValueStopwatch.StartNew(); - int outstanding; - while ((outstanding = GetOutstandingCount()) > 0 && watch.ElapsedMilliseconds < drainTimeout.TotalMilliseconds) + while (HasCallerWork() && watch.ElapsedMilliseconds < drainTimeout.TotalMilliseconds) { await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } - if (outstanding > 0) + if (HasCallerWork()) { - // deliberately reported: an abandoned command is exactly what a caller will be asking about - log?.LogInformationRetiringServerAbandoned(new(EndPoint), outstanding); + // deliberately reported: an abandoned command is exactly what a caller will be asking about. + // The count is the total, since that is what is actually being dropped on the floor + log?.LogInformationRetiringServerAbandoned(new(EndPoint), GetOutstandingCount()); } Dispose(); @@ -926,6 +946,11 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; + + if (connection is not null && bridge == interactive) + { + ReconcileMaintenanceNotifications(connection); + } if (bridge == subscription || isResp3) { // Note: this MUST be fire and forget, because we might be in the middle of a Sync processing @@ -1241,6 +1266,10 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) // forget what the previous connection's HELLO told us; re-established below, if this one repeats it // (the subscription handshake is deliberately left out of this: it doesn't do the discovery step) RoleKnownFromHello = false; + + // likewise per-connection: re-armed from this handshake's reply, if we ask + _maintenanceNotificationsActive = _maintenanceNotificationsRequested = false; + _maintenanceNotificationsRefusal = null; } // HELLO serves two purposes: negotiating RESP3, and reporting details we would otherwise need INFO or @@ -1332,6 +1361,19 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) msg = Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.ID); msg.SetInternalCall(); await WriteDirectOrQueueFireAndForgetAsync(connection, msg, autoConfig ??= ResultProcessor.AutoConfigureProcessor.Create(log)).ForAwait(); + + if (ShouldRequestMaintenanceNotifications(isInteractive, negotiateResp3)) + { + _maintenanceNotificationsRequested = true; + // speculative in the same way as the AUTH above: we don't yet know what HELLO negotiated, + // so we ask whenever we asked for RESP3, and ReconcileMaintenanceNotifications sorts out a + // downgrade once the reply has been processed. A bare ON is explicitly valid: the server + // then picks the endpoint type, which is what we want until we derive one ourselves. + log?.LogInformationRequestingMaintenanceNotifications(new(this), MaintenanceMode); + msg = Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON); + msg.SetInternalCall(); + await WriteDirectOrQueueFireAndForgetAsync(connection, msg, ResultProcessor.MaintenanceNotifications).ForAwait(); + } } var bridge = connection.BridgeCouldBeNull; diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 587af0622..e799f510a 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -42,6 +42,7 @@ + @@ -59,6 +60,7 @@ + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs new file mode 100644 index 000000000..b904632e2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The OSS cluster notification family against a real cluster: SMIGRATING, SMIGRATED, and what +/// the client is supposed to do about them. +/// +/// +/// Every /slot-migrate trigger requires oss_cluster: true with all-master-shards, so this +/// scenario family is the cluster half of the feature - the half whose reactions (re-reading the slot map, +/// re-subscribing stranded sharded channels) have until now only been exercised against the in-process fake. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "cluster-family")] +public class ClusterFamilyScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + /// + /// Slot migrations, in the four shapes the injector can produce. + /// + /// + /// The effects differ in what happens to the *endpoint list*, which is the thing a client has to keep up + /// with: remove-add retires a node and introduces one, remove only retires, add only + /// introduces, and slot-shuffle moves slots between nodes that both stay. So they cover the four + /// ways a slot map can go stale, and only one of them (shuffle) leaves the endpoint set alone. + /// + /// + /// Only remove here. The other effects need a database the setup leg will not provision, and get + /// their own fixture below rather than skipping: add and slot-shuffle need a node holding + /// several shards, and remove-add is not in /slot-migrate/setup's effect enum at all. + /// + [Theory] + [InlineData("remove", "migrate")] + public async Task SlotMigrationIsAnnouncedAndActedOn(string effect, string trigger) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "slot-migrate", effect, trigger, log.WriteLine, + setupTrigger: "reshard", cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + ScenarioSupport.RequireEffectIsAchievable(scenario, effect); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + foreach (var migration in push.SlotMigrations) + { + log.WriteLine($" slots {migration.RawSlots}: {migration.Source} -> {migration.Target}"); + } + } + }; + + var endpointsBefore = conn.GetEndPoints().Length; + log.WriteLine($"connected as {conn.GetServer(conn.GetEndPoints()[0]).ServerType} across {endpointsBefore} endpoint(s)"); + + // keys spread across slots, so a migration of any shard moves something we are actually using + var db = conn.GetDatabase(); + var keys = Enumerable.Range(0, 32).Select(i => (RedisKey)$"fi-{effect}-{i}").ToArray(); + foreach (var key in keys) await db.StringSetAsync(key, "before"); + + clock.Restart(); + await ScenarioSupport.FireOrSkipAsync(scenario, effect, cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports finished"); + + // Keep using the connection while things move: a slot map that has gone stale shows up as MOVED + // redirections, and the point of the feature is that we learn the new one rather than eating them. + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + int reads = 0, failures = 0; + while (clock.Elapsed < deadline) + { + foreach (var key in keys) + { + try + { + await db.StringGetAsync(key); + reads++; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + failures++; + if (failures <= 3) log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s read failed: {ex.Message}"); + } + } + + await Task.Delay(500, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s); {reads} reads, {failures} failures; " + + $"endpoints {endpointsBefore} -> {conn.GetEndPoints().Length}"); + + // A slot migration is announced to every proxy, so any connection should see it. What is asserted is + // only that we were told and understood it - the *reaction* is asserted separately below, because a + // migration that moves no slot we hold would legitimately change nothing here. + Assert.NotEmpty(events); + Assert.All(events, e => Assert.NotEqual(MaintenanceNotificationType.None, e.NotificationType)); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.SlotMigrating or MaintenanceNotificationType.SlotMigrated); + + // Any SMIGRATED that carried a payload must have parsed into something usable; an empty list would + // mean we saw the frame and threw the contents away. + foreach (var migrated in events.Where(e => e.NotificationType == MaintenanceNotificationType.SlotMigrated)) + { + if (!string.IsNullOrEmpty(migrated.Payload) || migrated.SlotMigrations.Count > 0) + { + Assert.NotEmpty(migrated.SlotMigrations); + Assert.All(migrated.SlotMigrations, m => Assert.NotNull(m.Source)); + } + } + } + + // and the client is still serving keys across the new topology + Assert.True(reads > 0, "no reads succeeded at all during the migration"); + foreach (var key in keys) Assert.Equal("before", await db.StringGetAsync(key)); + } + + [Fact] + public async Task ShardedSubscriptionSurvivesASlotMigration() + { + // D5's resubscription, against a real cluster. A sharded channel is bound to the node owning its slot, + // so a migration strands the subscription: the server sends an unsolicited SUNSUBSCRIBE and stops + // delivering. Recovering that is the client's job, and until now only the fake has tested it. + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "slot-migrate", "remove", "migrate", log.WriteLine, + setupTrigger: "reshard", cancellationToken: cancellationToken); + Assert.NotNull(scenario.Database); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(scenario.Database.GetClientConfig(fixture.Environment)); + var subscriber = conn.GetSubscriber(); + + // several channels, because the migration moves the slots of one node: with one channel the test would + // usually be asserting that nothing happened to it + var channels = Enumerable.Range(0, 16).Select(i => RedisChannel.Sharded($"fi-shard-{i}")).ToArray(); + var received = new int[channels.Length]; + for (int i = 0; i < channels.Length; i++) + { + var index = i; + await subscriber.SubscribeAsync(channels[i], (_, _) => Interlocked.Increment(ref received[index])); + } + + Assert.True(await DeliversEverywhereAsync(subscriber, channels, received, cancellationToken), + "every sharded channel should deliver before the migration, or the test proves nothing"); + log.WriteLine($"all {channels.Length} sharded channels delivering before the migration"); + + await ScenarioSupport.FireOrSkipAsync(scenario, "remove", cancellationToken); + + // The recovery is deliberately allowed to be slow: it is driven by a jittered topology refresh and, as a + // fallback, by a delayed resubscription sweep. What matters is that it happens without the caller doing + // anything, not that it is instant. + var recovered = await DeliversEverywhereAsync(subscriber, channels, received, cancellationToken, timeoutSeconds: 90); + log.WriteLine($"delivery after migration: {(recovered ? "all channels" : "INCOMPLETE")}"); + Assert.True(recovered, "sharded subscriptions should deliver again once the topology settles"); + } + + /// + /// Publishes to every channel and waits until each one has delivered at least one more message. + /// + /// + /// Publishing repeatedly rather than once: the publish itself routes by slot, so during a migration an + /// individual publish can land on a node that no longer owns the slot. Retrying is what a real caller would + /// do, and it keeps the test measuring *delivery* rather than the timing of one message. + /// + private async Task DeliversEverywhereAsync( + ISubscriber subscriber, + RedisChannel[] channels, + int[] received, + CancellationToken cancellationToken, + int timeoutSeconds = 20) + { + var baseline = channels.Select((_, i) => Volatile.Read(ref received[i])).ToArray(); + var deadline = DateTime.UtcNow.AddSeconds(timeoutSeconds); + + while (DateTime.UtcNow < deadline) + { + for (int i = 0; i < channels.Length; i++) + { + if (Volatile.Read(ref received[i]) > baseline[i]) continue; + try + { + await subscriber.PublishAsync(channels[i], "ping"); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // expected mid-migration; the retry is the point + } + } + + await Task.Delay(500, cancellationToken); + if (channels.Select((_, i) => Volatile.Read(ref received[i]) > baseline[i]).All(x => x)) return true; + } + + var missing = channels.Where((_, i) => Volatile.Read(ref received[i]) <= baseline[i]).Select(c => c.ToString()); + log.WriteLine($" not delivering: {string.Join(", ", missing)}"); + return false; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs new file mode 100644 index 000000000..2146492a5 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// An OSS cluster with packed shards, so add and slot-shuffle have something to move. +/// +public sealed class DenseClusterFixture() : FaultInjectorFixture(DatabaseShape.OssClusterDense); + +/// +/// The slot migrations the scenario setup legs cannot provision for. +/// +/// +/// These looked like an environment limitation - the injector refuses them with "No node with multiple shards +/// found" - and they are not: the setup leg provisions one shard per node, so there is never a node to take a +/// shard *from*. Six shards with dense placement over three nodes gives two per node, and both effects +/// then run. The lesson generalises: a scenario that setup cannot arrange is still reachable by provisioning +/// the database ourselves and driving the run leg by bdb_id. +/// +/// No scenario teardown here, deliberately. Teardown deletes the database, and this database belongs to the +/// fixture, which deletes it itself; and migrate excludes no nodes, so there is nothing to restore. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "cluster-family")] +public class DenseClusterScenarioTests(DenseClusterFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("add")] + [InlineData("slot-shuffle")] + [InlineData("remove-add")] + public async Task SlotMigrationIsAnnouncedAndActedOn(string effect) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + log.WriteLine($"{effect} against {database}"); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + foreach (var migration in push.SlotMigrations) + { + log.WriteLine($" slots {migration.RawSlots}: {migration.Source} -> {migration.Target}"); + } + } + }; + + var db = conn.GetDatabase(); + var keys = Enumerable.Range(0, 32).Select(i => (RedisKey)$"fi-dense-{effect}-{i}").ToArray(); + foreach (var key in keys) await db.StringSetAsync(key, "before"); + + clock.Restart(); + var query = new Dictionary + { + ["effect"] = effect, + ["trigger"] = "migrate", + ["bdb_id"] = database.BdbId.ToString(), + }; + await fixture.Injector.PostScenarioAsync("slot-migrate", leg: null, query, cancellationToken: cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports finished"); + + int reads = 0, failures = 0; + var deadline = clock.Elapsed + TimeSpan.FromSeconds(20); + while (clock.Elapsed < deadline) + { + foreach (var key in keys) + { + try + { + await db.StringGetAsync(key); + reads++; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + failures++; + } + } + + await Task.Delay(500, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s); {reads} reads, {failures} failures"); + Assert.NotEmpty(events); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.SlotMigrating + or MaintenanceNotificationType.SlotMigrated); + } + + Assert.True(reads > 0, "no reads succeeded during the migration"); + foreach (var key in keys) Assert.Equal("before", await db.StringGetAsync(key)); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs new file mode 100644 index 000000000..de33a1d44 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Security.Cryptography.X509Certificates; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Checks that the environment's TLS material belongs to the cluster we are about to talk to. +/// +/// +/// Environments get re-provisioned, and the certificates do not always follow. Observed 2026-09-01: the folder +/// held a server certificate for *.marcgravell-test-46be1d08... while the live cluster was +/// marcgravell-test-e21cd75d... - a leftover from a previous provision, three days older than the +/// env_output.json beside it. +/// +/// Without this check, that presents as a TLS handshake failure, which reads like a client bug and is not one: +/// the certificate genuinely does not cover the name being dialled, and refusing it is correct +/// (TrustIssuer tolerates chain errors only, so a name mismatch fails - as it should). Detecting it here +/// turns half an hour of certificate archaeology into a skip that names both clusters, and it costs nothing +/// because it happens before a database is provisioned. +/// +/// +internal static class CertificateSanity +{ + /// + /// Skips when the environment's certificates were issued for a different cluster. + /// + public static void RequireCertificatesMatchThisCluster(FaultInjectorEnvironment environment, Action log) + { + var clusterName = environment.Cluster?.ClusterName; + if (clusterName is null) return; // nothing to compare against; let the connect speak for itself + + // the server certificate the environment generated, if it left one behind + var leafPath = Path.Combine(environment.ConfigDirectory.FullName, "redis.crt"); + if (!File.Exists(leafPath)) return; + + var leaf = X509CertificateLoader.LoadCertificateFromFile(leafPath); + var names = ReadDnsNames(leaf); + log($"environment server certificate covers [{string.Join(", ", names)}]; cluster is '{clusterName}'"); + + if (!names.Any(name => CoversClusterEndpoints(name, clusterName))) + { + Assert.Skip( + $"the environment's TLS certificates cover [{string.Join(", ", names)}] but this cluster is " + + $"'{clusterName}' - they are left over from an earlier provision, so a TLS test here would " + + "only be measuring the mismatch. Re-provision with certificate generation enabled to run it."); + } + } + + /// + /// Every DNS name a certificate carries, not just the first. + /// + /// + /// GetNameInfo returns one name, which is not enough: these certificates carry both a wildcard and + /// the bare cluster name, and which one comes back decides the answer. Reading the SAN extension properly is + /// the difference between a check that works and one that skips a perfectly good environment - as the first + /// version of this did. + /// + private static List ReadDnsNames(X509Certificate2 certificate) + { + foreach (var extension in certificate.Extensions) + { + if (extension is X509SubjectAlternativeNameExtension san) + { + return [.. san.EnumerateDnsNames()]; + } + } + + var subject = certificate.GetNameInfo(X509NameType.DnsName, forIssuer: false); + return subject is null ? [] : [subject]; + } + + /// + /// Whether a certificate name covers the endpoints of a given cluster. + /// + /// + /// The hosts actually dialled are *database* endpoints - redis-13500.<cluster> - so a wildcard + /// whose parent is the cluster name is exactly right, even though (correctly) it does not match the bare + /// cluster name itself. An exact SAN for the cluster name also counts, since that is what the REST API is + /// reached by. + /// + private static bool CoversClusterEndpoints(string certificateName, string clusterName) + { + if (certificateName.StartsWith("*.", StringComparison.Ordinal)) + { + return string.Equals(certificateName[2..], clusterName, StringComparison.OrdinalIgnoreCase); + } + + return string.Equals(certificateName, clusterName, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs new file mode 100644 index 000000000..c9565bb79 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The cluster's own REST API on port 9443, for the few facts the fault injector does not expose. +/// +/// +/// Reads only. Anything that *changes* state goes through the injector so it is recorded as a job with an +/// action id - which is also how the console works, and it means a scenario can be reconstructed afterwards +/// from the injector's history rather than from somebody's memory. +/// +public sealed class ClusterRestClient : IDisposable +{ + private readonly HttpClient _http; + + public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, string? certificateAuthorityPath) + { + var handler = new HttpClientHandler(); + + if (certificateAuthorityPath is not null) + { + // The cluster's management certificate is self-signed per environment, like the proxy ones. Pin to + // the CA in the config directory rather than accepting anything: this channel carries credentials. + var issuer = X509CertificateLoader.LoadCertificateFromFile(certificateAuthorityPath); + handler.ServerCertificateCustomValidationCallback = (_, certificate, chain, _) => + { + if (certificate is null || chain is null) return false; + chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.Add(issuer); + chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + return chain.Build(certificate); + }; + } + + _http = new HttpClient(handler) { BaseAddress = credentials.RestUrl, Timeout = TimeSpan.FromSeconds(30) }; + var basic = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credentials.Username}:{credentials.Password}")); + _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basic); + } + + public void Dispose() => _http.Dispose(); + + /// + /// Every database on the cluster, as (bdb_id, name). + /// + public async Task> ListDatabasesAsync() + { + using var response = await _http.GetAsync("/v1/bdbs?fields=uid,name"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var results = new List<(int, string)>(); + foreach (var bdb in document.RootElement.EnumerateArray()) + { + if (bdb.TryGetProperty("uid", out var uid) && bdb.TryGetProperty("name", out var name) + && uid.TryGetInt32(out var id) && name.GetString() is { } text) + { + results.Add((id, text)); + } + } + + return results; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs new file mode 100644 index 000000000..b9778e213 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The database shapes this suite provisions, described declaratively. +/// +/// +/// Shape is the axis that matters, and it is not cosmetic: measurements on 2026-08-28 showed the number of A +/// records a hostname carries follows actual proxy *placement*, giving 2 for all-nodes, 3 for +/// all-master-shards and 1 for single - and the handoff takes a different branch depending on +/// whether a live sibling address exists. So the shapes below are the reason to provision databases from inside +/// the tests at all: they turn that into a matrix axis instead of a manual sweep. +/// +public sealed record DatabaseShape( + string Label, + bool OssCluster = false, + string? ProxyPolicy = null, + bool Tls = false, + bool Replication = false, + int ShardCount = 1, + string? EndpointType = null, + string ShardsPlacement = "sparse", + string IpType = "external") +{ + /// A proxied standalone database: the shape MOVING actually fires on. + public static readonly DatabaseShape ProxiedStandalone = new("proxied-standalone", ProxyPolicy: "single"); + + /// Multiple proxies, so the hostname carries several A records and a sibling always exists. + public static readonly DatabaseShape MultiProxy = new("multi-proxy", ProxyPolicy: "all-master-shards", ShardCount: 2); + + /// OSS cluster API: the family that emits SMIGRATING/SMIGRATED instead. + public static readonly DatabaseShape OssClusterApi = new("oss-cluster", OssCluster: true, ProxyPolicy: "all-master-shards", ShardCount: 2); + + /// + /// An OSS cluster whose shards are packed, so some node holds more than one. + /// + /// + /// Exists to reach the add and slot-shuffle migrations, which need a node holding several + /// shards to have something to move. The scenario setup legs cannot produce that - they provision one shard + /// per node - so those effects looked like an environment limitation until this was tried: six shards with + /// dense placement over three nodes gives two per node, and both effects then run. + /// + /// Note it also demonstrates the placement-versus-policy point from the other direction: packed masters mean + /// all-master-shards advertises a *single* address, because the count follows placement. + /// + /// + public static readonly DatabaseShape OssClusterDense = new( + "oss-cluster-dense", OssCluster: true, ProxyPolicy: "all-master-shards", ShardCount: 6, ShardsPlacement: "dense"); + + /// + /// TLS with hostname-advertised endpoints - the documented coverage gap. + /// + /// + /// endpoint_type: ip is the interesting counterpart: with addresses advertised, a verifying client + /// cannot check identity on the targets it is told to use, because the proxy certificate carries DNS names + /// and no IP SAN. That pairing is the thing no in-process harness can honestly reproduce. + /// + public static readonly DatabaseShape TlsWithHostnames = new("tls-hostnames", OssCluster: true, Tls: true, ShardCount: 2, EndpointType: "hostname"); + + /// + /// The create_database parameters for this shape. + /// + /// + /// Goes inside a database_config wrapper when it reaches the injector - a flat payload is rejected + /// with "Invalid parameter 'database_config': got None". + /// + /// Built as data rather than as a typed record, because these are the injector's wire names and its schema + /// declares parameters as untyped. The names here are no longer guesses: they match the environment's + /// own bdb_config.json and the dbconfig the injector itself publishes as a trigger requirement + /// (GET /topology-change-standalone?effect=...), which is the authoritative list. + /// + /// + public Dictionary ToCreateParameters(string name, int port) + { + var parameters = new Dictionary + { + ["name"] = name, + ["port"] = port, + ["memory_size"] = 134_217_728, // 128MB, matching what the injector asks for in its own requirements + ["eviction_policy"] = "volatile-lru", + ["replication"] = Replication, + ["sharding"] = ShardCount > 1, + ["shards_count"] = ShardCount, + ["shards_placement"] = ShardsPlacement, + ["oss_cluster"] = OssCluster, + }; + + if (ShardCount > 1) + { + // Required whenever sharding is on: Redis Enterprise rejects the database outright with + // "Invalid sharding configuration: missing shard_key_regex". These two patterns are the standard + // pair - an explicit hash tag if present, otherwise the whole key - and are what the environment's + // own bdb_config.json uses. + parameters["shard_key_regex"] = new[] + { + new Dictionary { ["regex"] = ".*\\{(?.*)\\}.*" }, + new Dictionary { ["regex"] = "(?.*)" }, + }; + } + + // only when asked for: a database with no tls_mode serves plaintext, and setting it here without + // certificates in place produces a database nothing can connect to + if (Tls) parameters["tls_mode"] = "enabled"; + + if (OssCluster) + { + // Which addresses CLUSTER SLOTS advertises, and it is not cosmetic: the default is *internal*, so a + // cluster database created without this advertises VPC-private addresses. A client outside the VPC + // then connects to the seed, discovers nodes at 10.x, and cannot reach any of them - which is + // exactly how the first dense-cluster run failed. Distinct from EndpointType (ip versus hostname), + // which is about identity rather than routing. + parameters["oss_cluster_api_preferred_ip_type"] = IpType; + } + + if (OssCluster && EndpointType is not null) + { + // Two distinct axes, easily conflated. "endpoint_type" is ip-versus-hostname - what CLUSTER SLOTS + // advertises, and therefore whether a verifying TLS client can check identity on the targets it is + // given. "ip_type" is internal-versus-external, which is about routing rather than identity. + parameters["oss_cluster_api_preferred_endpoint_type"] = EndpointType; + } + + if (ProxyPolicy is not null) parameters["proxy_policy"] = ProxyPolicy; + + return parameters; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs new file mode 100644 index 000000000..3040ea891 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A database the environment already provisioned, read from endpoints.json. +/// +/// +/// The counterpart to creating our own. Provisioning gives control over the shape, which is what the matrix +/// needs; this gives a run against whatever the environment template made, which is what you want when the +/// question is "does any of this work at all" rather than "does it work for shape X". +/// +/// Note endpoints.json carries more than expected: each entry's raw_endpoints includes +/// proxy_policy, oss_cluster_api_preferred_endpoint_type and the address list behind the DNS +/// name. So the facts that decide client behaviour are mostly here, and the cluster REST API is only needed for +/// the explicit oss_cluster flag and the client-certificate settings. +/// +/// +public sealed record ExistingDatabase( + string Key, + int BdbId, + string Host, + int Port, + bool Tls, + string? Username, + string? Password, + string? ProxyPolicy, + string? EndpointType, + IReadOnlyList Addresses) +{ + /// + /// How many addresses the hostname is expected to resolve to. + /// + /// + /// The measured driver of handoff behaviour: with more than one address a live sibling always exists, so a + /// handoff steps sideways immediately; with one, it has to wait for the record to move. Tests that care + /// should assert on this rather than on the policy name, because the count follows actual proxy + /// *placement* - an all-master-shards database whose shards share a node advertises one address. + /// + public int AdvertisedAddressCount => Addresses.Count; + + public override string ToString() => $"{Key} ({Host}:{Port}, bdb {BdbId}, {ProxyPolicy ?? "?"}, {AdvertisedAddressCount} addr)"; + + /// + /// Reads every database in the environment's endpoints.json, keyed as that file keys them. + /// + public static Dictionary ReadAll(FaultInjectorEnvironment environment) + { + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + var path = Path.Combine(environment.ConfigDirectory.FullName, "endpoints.json"); + if (!File.Exists(path)) return results; + + using var document = JsonDocument.Parse(File.ReadAllText(path)); + foreach (var entry in document.RootElement.EnumerateObject()) + { + if (entry.Value.ValueKind != JsonValueKind.Object) continue; + if (TryRead(entry.Name, entry.Value, out var database)) results[entry.Name] = database; + } + + return results; + } + + private static bool TryRead(string key, JsonElement element, out ExistingDatabase database) + { + database = null!; + if (!element.TryGetProperty("bdb_id", out var bdbId) || !bdbId.TryGetInt32(out var id)) return false; + + // "endpoints" holds "host:port" or a redis:// URI depending on how the environment was templated + string? host = null; + int port = 0; + if (element.TryGetProperty("endpoints", out var endpoints) && endpoints.ValueKind == JsonValueKind.Array + && endpoints.GetArrayLength() > 0 && endpoints[0].GetString() is { } first) + { + var text = first; + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) text = text[(scheme + 3)..]; + var colon = text.LastIndexOf(':'); + if (colon > 0 && int.TryParse(text[(colon + 1)..], out port)) host = text[..colon]; + } + + string? proxyPolicy = null, endpointType = null; + var addresses = new List(); + if (element.TryGetProperty("raw_endpoints", out var raw) && raw.ValueKind == JsonValueKind.Array + && raw.GetArrayLength() > 0) + { + var head = raw[0]; + proxyPolicy = ReadString(head, "proxy_policy"); + endpointType = ReadString(head, "oss_cluster_api_preferred_endpoint_type"); + host ??= ReadString(head, "dns_name"); + if (port == 0 && head.TryGetProperty("port", out var rawPort)) rawPort.TryGetInt32(out port); + + if (head.TryGetProperty("addr", out var addr) && addr.ValueKind == JsonValueKind.Array) + { + foreach (var item in addr.EnumerateArray()) + { + if (item.GetString() is { } address) addresses.Add(address); + } + } + } + + if (host is null || port == 0) return false; + + database = new ExistingDatabase( + key, + id, + host, + port, + element.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, + ReadString(element, "username"), + ReadString(element, "password"), + proxyPolicy, + endpointType, + addresses); + return true; + } + + private static string? ReadString(JsonElement element, string name) + => element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null; + + /// + /// Connection options for this database, with maintenance notifications required. + /// + public ConfigurationOptions GetClientConfig(FaultInjectorEnvironment environment, MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + User = string.Equals(Username, "default", StringComparison.OrdinalIgnoreCase) ? null : Username, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = mode, + AbortOnConnectFail = false, + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Tls) + { + options.Ssl = true; + options.SslHost = Host; + var caPath = environment.CertificateAuthorityPath + ?? throw new InvalidOperationException($"{Key} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}"); + options.TrustIssuer(caPath); + } + + return options; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs new file mode 100644 index 000000000..c6ebe4453 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Runs against the databases the environment template already created, rather than provisioning new ones. +/// +/// +/// The cheap tier, and the right one to start from: it needs none of the create_database schema, so it +/// answers "does the client work against a real deployment" before anything depends on parameter names that are +/// documented only as prose. +/// +public class ExistingDatabaseFixture : IAsyncLifetime +{ + private FaultInjectorClient? _injector; + private string? _skipReason; + + public IReadOnlyDictionary Databases { get; private set; } + = new Dictionary(); + + public FaultInjectorEnvironment Environment => + FaultInjectorEnvironment.Current ?? throw new InvalidOperationException("no environment; call RequireAvailable first"); + + public FaultInjectorClient Injector => + _injector ?? throw new InvalidOperationException("no injector; call RequireAvailable first"); + + /// Skips the calling test when there is nothing to talk to. + public void RequireAvailable() + { + if (_skipReason is not null) Assert.Skip(_skipReason); + } + + /// + /// The named database, or a skip if this environment does not have one. + /// + /// + /// A skip rather than a failure, deliberately: which databases exist is a property of the environment + /// template somebody chose, so a template without an oss_cluster database is a reason not to run the + /// cluster tests, not evidence of a bug. + /// + public ExistingDatabase Require(string key) + { + RequireAvailable(); + if (!Databases.TryGetValue(key, out var database)) + { + Assert.Skip($"this environment has no '{key}' database (found: {string.Join(", ", Databases.Keys)})"); + } + + return database!; + } + + /// + /// Where fixture-level diagnostics go. Deliberately collected rather than written to + /// : a fixture has no test output helper, and anything it prints to the console is + /// easily lost - which is how a silently failing setup call survived a day of use. + /// + public List SetupLog { get; } = []; + + private void log(string message) + { + SetupLog.Add(message); + Console.WriteLine(message); + } + + public async ValueTask InitializeAsync() + { + if (FaultInjectorEnvironment.Current is null) + { + _skipReason = FaultInjectorEnvironment.UnavailableReason ?? "no fault-injector environment configured"; + return; + } + + if (!FaultInjectorEnvironment.IsEnabled) + { + _skipReason = "set E2E_SCENARIO_TESTS=true to run against a real deployment"; + return; + } + + _injector = new FaultInjectorClient(FaultInjectorEnvironment.Current.InjectorUrl); + Databases = ExistingDatabase.ReadAll(FaultInjectorEnvironment.Current); + await EnsureMaintenanceNotificationsEnabledAsync(); + } + + public ValueTask DisposeAsync() + { + _injector?.Dispose(); + return default; + } + + /// + /// Turns on the cluster-level maintenance-notification flags. + /// + /// + /// A different thing from the per-connection opt-in, and both are required: the cluster flag decides whether + /// CLIENT MAINT_NOTIFICATIONS exists at all, and the command opts one connection in. Without this a + /// run fails at connect - correctly, since the tests ask for + /// - but for a reason that has nothing to do with the + /// client, so it is worth setting rather than diagnosing. + /// + /// Through the injector rather than a direct REST PUT, so the change leaves an action id behind and + /// shows up in the injector's history like everything else. Best-effort: if the action type is not + /// available on this build, the connect failure that follows says so clearly enough. + /// + /// + private async Task EnsureMaintenanceNotificationsEnabledAsync() + { + // nested under "config", not flattened - the same shape create_database wants for "database_config". + // A flat payload is rejected with "Invalid parameter 'config': got None", and because this call is + // best-effort it failed *silently* for a whole day of testing without anybody noticing. The impact was + // small: the environment templates enable these flags at provision time, so this is a safety net rather + // than the mechanism, and the tests were passing on their own merits. It matters for an environment + // provisioned without them, where the alternative is every test failing at connect and blaming the + // client for a server-side setting. + var parameters = new Dictionary + { + ["config"] = new Dictionary + { + ["client_maint_notifications"] = true, // proxy-routed databases + ["oss_cluster_client_maint_notifications"] = true, // oss_cluster databases + }, + }; + + try + { + await Injector.RunActionAsync("update_cluster_config", parameters, timeout: TimeSpan.FromMinutes(2)); + log("cluster maintenance-notification flags enabled"); + } + catch (Exception ex) + { + // Still best-effort - a cluster may not support the flags at all - but no longer silent: it goes to + // the test output, where the connect failure that follows can be attributed to it. + log($"could not set cluster maintenance-notification flags: {ScenarioSupport.Summarize(ex.Message)}"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs new file mode 100644 index 000000000..6b1b18b5a --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Everything this suite needs to reach a real deployment, discovered from one directory. +/// +/// +/// One path is the whole configuration, deliberately. That directory is the one mounted into the fault +/// injector as /app/config and the one docker compose up is run from, so it already holds the +/// compose file, the cluster credentials (env_output.json), the CA certificate, and - once databases +/// exist - endpoints.json. Asking for anything else would mean the person who provisioned the +/// environment has to hand-carry facts into the test run, which is the error-prone part. +/// +/// Point at it with SER_FI_CONFIG_DIR; FI_CONSOLE_CONFIG_DIR is honoured as a fallback so this +/// runs against the same directory as the fault-injector console with no extra setup. +/// +/// +public sealed class FaultInjectorEnvironment +{ + private const string ConfigDirVariable = "SER_FI_CONFIG_DIR"; + private const string ConsoleConfigDirVariable = "FI_CONSOLE_CONFIG_DIR"; + private const string InjectorUrlVariable = "FAULT_INJECTION_API_URL"; + private const string EnabledVariable = "E2E_SCENARIO_TESTS"; + private const string DefaultInjectorUrl = "http://127.0.0.1:20324"; + + /// + /// The environment for this run, or null when none is configured. + /// + public static FaultInjectorEnvironment? Current { get; } = Discover(); + + /// + /// Why there is no environment, for a skip message that says something useful. + /// + public static string? UnavailableReason { get; private set; } + + private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) + { + ConfigDirectory = configDirectory; + InjectorUrl = injectorUrl; + } + + public DirectoryInfo ConfigDirectory { get; } + + public Uri InjectorUrl { get; } + + /// + /// The CA certificate that signs the proxy certificates, for . + /// + /// + /// The certificates are self-signed per environment, so trusting the issuer is how a TLS test validates + /// anything at all. Note what this is *not*: a switch that disables validation. A test that cannot find the + /// CA fails rather than falling back to trusting everything, because a TLS test that silently stops + /// checking identity is worse than no TLS test - it reports success for the one thing it exists to catch. + /// + public string? CertificateAuthorityPath { get; private set; } + + /// + /// Cluster credentials for the REST API on port 9443, when env_output.json carries them. + /// + /// + /// Needed for the facts endpoints.json does not carry - notably oss_cluster and the + /// advertised endpoint type - which matter because they decide which notification family a database can + /// even emit. Tests that provision their own database know what they asked for and need none of this. + /// + public ClusterCredentials? Cluster { get; private set; } + + private static FaultInjectorEnvironment? Discover() + { + var path = Environment.GetEnvironmentVariable(ConfigDirVariable) + ?? Environment.GetEnvironmentVariable(ConsoleConfigDirVariable); + + if (string.IsNullOrWhiteSpace(path)) + { + UnavailableReason = $"no fault-injector environment: set {ConfigDirVariable} to the directory holding env_output.json"; + return null; + } + + var directory = new DirectoryInfo(path); + if (!directory.Exists) + { + // configured but wrong is a mistake worth reporting, not a reason to quietly do nothing + UnavailableReason = $"{ConfigDirVariable} points at '{path}', which does not exist"; + return null; + } + + var url = Environment.GetEnvironmentVariable(InjectorUrlVariable); + var injectorUrl = Uri.TryCreate(url, UriKind.Absolute, out var parsed) ? parsed : new Uri(DefaultInjectorUrl); + + var result = new FaultInjectorEnvironment(directory, injectorUrl) + { + CertificateAuthorityPath = FindCertificateAuthority(directory), + }; + result.Cluster = ReadClusterCredentials(directory); + return result; + } + + /// + /// Whether the caller actually meant to run destructive tests against a real deployment. + /// + /// + /// Two gates rather than one, and they mean different things. Without a config directory there is nothing + /// to talk to, which is the ordinary state for everybody else and skips. With a directory but without + /// E2E_SCENARIO_TESTS=true we also skip, because these tests create and delete databases and nobody + /// should trip that by running the full traversal. What must *not* happen is a third state where the + /// environment is configured, meant, and broken, yet the run still reports success - see + /// . + /// + public static bool IsEnabled => + string.Equals(Environment.GetEnvironmentVariable(EnabledVariable), "true", StringComparison.OrdinalIgnoreCase); + + private static string? FindCertificateAuthority(DirectoryInfo directory) + { + // names seen across the environment templates, most specific first + foreach (var candidate in new[] { "ca.pem", "ca.crt", "proxy_cert.pem", "redislabs_ca.pem" }) + { + var path = Path.Combine(directory.FullName, candidate); + if (File.Exists(path)) return path; + } + + return null; + } + + private static ClusterCredentials? ReadClusterCredentials(DirectoryInfo directory) + { + var path = Path.Combine(directory.FullName, "env_output.json"); + if (!File.Exists(path)) return null; + + try + { + using var document = JsonDocument.Parse(File.ReadAllText(path)); + + // two schemas in the wild: single-cluster templates put outputs at the top level, and the AWS + // multi-cluster template nests them under .clusters.value[N]. Both are handled because which one + // you get is a property of the template somebody chose, not of anything a test can control. + var root = document.RootElement; + if (root.TryGetProperty("clusters", out var clusters) + && clusters.TryGetProperty("value", out var values) + && values.ValueKind == JsonValueKind.Array + && values.GetArrayLength() > 0) + { + root = values[0]; + } + + var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); + var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); + var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + + return name is null || user is null || password is null ? null : new ClusterCredentials(name, user, password); + } + catch (Exception) + { + // a malformed env_output.json is not fatal here: only the tests that need REST enrichment care, + // and they report it themselves rather than failing every test in the suite at discovery time + return null; + } + } + + /// + /// Reads a property that may be a bare value or a terraform-style { "value": ... } wrapper. + /// + private static string? ReadValue(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var value)) return null; + if (value.ValueKind == JsonValueKind.Object && value.TryGetProperty("value", out var inner)) value = inner; + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + public sealed record ClusterCredentials(string ClusterName, string Username, string Password) + { + public Uri RestUrl => new($"https://{ClusterName}:9443"); + + /// Never let the password reach a log; the endpoints are real and reachable. + public override string ToString() => $"{Username}@{ClusterName}"; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs new file mode 100644 index 000000000..6406b8b88 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Provisions one database shape for the tests that need it, and takes it away afterwards. +/// +/// +/// Per *shape*, not per test: creating a database on a real cluster is slow, so several test classes share one. +/// +/// The three-state behaviour matters more than the provisioning. No environment configured, or the +/// E2E_SCENARIO_TESTS gate not set, means every test skips - which is the ordinary case for anybody +/// running the full traversal, and these tests create and delete real databases. But an environment that *is* +/// configured and meant, and then does not work, must **fail**: a suite that skips on a broken environment +/// reports success for tests that never ran, and you will trust it exactly when you should not. So the absent +/// case skips from the test body, and the broken case throws from here. +/// +/// +public abstract class FaultInjectorFixture(DatabaseShape shape) : IAsyncLifetime +{ + /// + /// Shared by every database this suite creates, in every run. + /// + /// + /// The sweep that cleans up leaks from earlier runs matches on this and nothing else, so it can never touch + /// a database somebody created by hand. Worth being strict about: the alternative is a test suite that + /// deletes production-shaped things on a shared cluster. + /// + public const string NamePrefix = "sertest-"; + + private static readonly string RunId = Guid.NewGuid().ToString("n")[..6]; + + private FaultInjectorClient? _injector; + private string? _skipReason; + + public DatabaseShape Shape { get; } = shape; + + /// The database created for this fixture, once has run. + public ProvisionedDatabase? Database { get; private set; } + + public FaultInjectorEnvironment Environment => + FaultInjectorEnvironment.Current ?? throw new InvalidOperationException("no environment; call RequireAvailable first"); + + public FaultInjectorClient Injector => + _injector ?? throw new InvalidOperationException("no injector; call RequireAvailable first"); + + /// + /// Skips the calling test when there is no environment to talk to. + /// + public void RequireAvailable() + { + if (_skipReason is not null) Assert.Skip(_skipReason); + } + + public async ValueTask InitializeAsync() + { + if (FaultInjectorEnvironment.Current is null) + { + _skipReason = FaultInjectorEnvironment.UnavailableReason ?? "no fault-injector environment configured"; + return; + } + + if (!FaultInjectorEnvironment.IsEnabled) + { + _skipReason = "set E2E_SCENARIO_TESTS=true to run against a real deployment (these tests create and delete databases)"; + return; + } + + // configured and meant: from here on, problems are failures + _injector = new FaultInjectorClient(FaultInjectorEnvironment.Current.InjectorUrl); + await SweepOrphansAsync(); + Database = await CreateDatabaseAsync(); + } + + public async ValueTask DisposeAsync() + { + // unconditionally, and before anything else can throw: a database left behind holds a port and a slice + // of cluster memory, and the next run collides with it + if (Database is { } database && _injector is { } injector) + { + try + { + await injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = database.BdbId }); + } + catch (Exception ex) + { + // never mask a test failure with a cleanup failure; the sweep will get it next time + Console.WriteLine($"failed to delete {database.Name} (bdb {database.BdbId}): {ex.Message}"); + } + } + + _injector?.Dispose(); + } + + /// + /// Creates the database for this shape, working around port collisions the way go-redis has to. + /// + /// + /// Port collisions are common enough on a shared cluster that go-redis carries a dedicated + /// CreateDatabaseWithPortRetry helper; this is the same idea. The base port is deliberately high and + /// the walk is upward, so a collision costs one attempt rather than a failed run. + /// + private async Task CreateDatabaseAsync() + { + const int BasePort = 13500, Attempts = 8; + var name = $"{NamePrefix}{Shape.Label}-{RunId}"; + Exception? last = null; + + for (int attempt = 0; attempt < Attempts; attempt++) + { + var port = BasePort + (attempt * 3); + try + { + // nested under database_config, not flattened: the injector answers a flat payload with + // "Invalid parameter 'database_config': got None" + var result = await Injector.RunActionAsync( + "create_database", + new Dictionary { ["database_config"] = Shape.ToCreateParameters(name, port) }); + return ProvisionedDatabase.FromCreateResult(name, port, Shape, result, Environment); + } + catch (Exception ex) when (IsPortCollision(ex)) + { + last = ex; + Console.WriteLine($"create_database on port {port} collided, retrying higher"); + } + } + + throw new InvalidOperationException($"could not create '{name}' after {Attempts} attempts", last); + } + + /// + /// Whether a create failure is worth trying a different port for. + /// + /// + /// Only port collisions are: everything else - a malformed config, a cluster with no capacity - fails + /// identically on every port, and retrying eight times turns one clear error message into eight and delays + /// the report by half a minute. The first version of this retried everything, and buried + /// "missing shard_key_regex" under eight identical tracebacks. + /// + private static bool IsPortCollision(Exception ex) => + ex.Message.Contains("port", StringComparison.OrdinalIgnoreCase) + && (ex.Message.Contains("in use", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("already", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("taken", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("conflict", StringComparison.OrdinalIgnoreCase)); + + /// + /// Best-effort removal of databases left behind by earlier runs. + /// + /// + /// Best-effort on purpose: listing databases needs the cluster REST API, which needs credentials this + /// directory may not carry, and being unable to tidy up is not a reason to fail a run that could otherwise + /// proceed. The per-fixture delete in is the reliable path; this only catches + /// leaks from a run that was killed. + /// + private async Task SweepOrphansAsync() + { + var cluster = Environment.Cluster; + if (cluster is null) + { + Console.WriteLine("no cluster credentials in env_output.json; skipping orphan sweep"); + return; + } + + try + { + using var rest = new ClusterRestClient(cluster, Environment.CertificateAuthorityPath); + foreach (var (bdbId, name) in await rest.ListDatabasesAsync()) + { + if (!name.StartsWith(NamePrefix, StringComparison.Ordinal)) continue; // never anything but ours + + Console.WriteLine($"sweeping orphaned test database {name} (bdb {bdbId})"); + await Injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = bdbId }); + } + } + catch (Exception ex) + { + Console.WriteLine($"orphan sweep skipped: {ex.Message}"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs new file mode 100644 index 000000000..a6268eb87 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A database this suite created, and how to connect to it. +/// +/// +/// Note what this type does *not* need to do: read endpoints.json to discover whether the database is +/// oss_cluster, or what endpoint type it advertises. A test that provisioned the database knows what it +/// asked for, which is the real argument for provisioning from inside the suite - the alternative is carrying +/// those facts from whoever set the environment up into the run, by hand. +/// +public sealed class ProvisionedDatabase +{ + private ProvisionedDatabase(string name, int bdbId, string host, int port, DatabaseShape shape, FaultInjectorEnvironment environment) + { + Name = name; + BdbId = bdbId; + Host = host; + Port = port; + Shape = shape; + Environment = environment; + } + + public string Name { get; } + + public int BdbId { get; } + + public string Host { get; } + + public int Port { get; } + + public DatabaseShape Shape { get; } + + public FaultInjectorEnvironment Environment { get; } + + public string? Password { get; private init; } + + /// + /// Connection options for this database, with maintenance notifications requested. + /// + /// + /// rather than Auto: in a test, a server that + /// silently declines the opt-in should fail the connection loudly rather than produce a run that passes + /// while observing nothing. That is the opposite of the right default for production, and exactly right + /// here. + /// + public ConfigurationOptions GetClientConfig() + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = MaintenanceNotificationMode.Enabled, + AbortOnConnectFail = false, + // real network, real cluster: connect and command budgets have to tolerate a WAN round trip, and a + // cluster that is mid-scenario is slower still + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Shape.Tls) + { + options.Ssl = true; + options.SslHost = Host; + + // The certificates are self-signed per environment, so trusting the issuer is what makes a TLS test + // mean anything. If the CA is missing we fail here rather than disabling validation: a TLS test that + // quietly stops checking identity reports success for the one thing it exists to catch. + var caPath = Environment.CertificateAuthorityPath + ?? throw new InvalidOperationException( + $"{Shape.Label} needs TLS, but no CA certificate was found in {Environment.ConfigDirectory.FullName}; " + + "validation is not disabled for tests"); + options.TrustIssuer(caPath); + } + + return options; + } + + /// Masked, because these endpoints are real and reachable from the internet. + public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, {Shape.Label})"; + + /// + /// Reads what the injector reported back after create_database. + /// + /// + /// Tolerant by design: the response shape is documented as prose, and the fields worth having may arrive at + /// the top level or nested under an output/result object. What cannot be guessed is the bdb_id - + /// without it there is nothing to delete afterwards - so that one is required and its absence is loud. + /// + public static ProvisionedDatabase FromCreateResult( + string name, + int requestedPort, + DatabaseShape shape, + JsonElement result, + FaultInjectorEnvironment environment) + { + var bdbId = FindInt(result, "bdb_id") + ?? throw new InvalidOperationException($"create_database for '{name}' returned no bdb_id: {result}"); + + var host = FindString(result, "endpoint") ?? FindString(result, "dns_name") ?? environment.Cluster?.ClusterName + ?? throw new InvalidOperationException($"create_database for '{name}' returned no endpoint, and env_output.json names no cluster: {result}"); + + // an endpoint may arrive as "host:port" or as a bare host + var port = requestedPort; + var colon = host.LastIndexOf(':'); + if (colon > 0 && int.TryParse(host[(colon + 1)..], out var parsedPort)) + { + port = parsedPort; + host = host[..colon]; + } + + return new ProvisionedDatabase(name, bdbId, host, port, shape, environment) + { + Password = FindString(result, "password"), + }; + } + + private static int? FindInt(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.Number && candidate.TryGetInt32(out var value)) return value; + if (candidate.ValueKind == JsonValueKind.String && int.TryParse(candidate.GetString(), out var parsed)) return parsed; + } + + return null; + } + + private static string? FindString(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.String) return candidate.GetString(); + if (candidate.ValueKind == JsonValueKind.Array && candidate.GetArrayLength() > 0 && candidate[0].ValueKind == JsonValueKind.String) + { + return candidate[0].GetString(); // endpoints arrive as a list often enough + } + } + + return null; + } + + /// + /// Yields every value for a property name, at any depth - shallowest first. + /// + private static IEnumerable Walk(JsonElement element, string name) + { + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty(name, out var direct)) yield return direct; + + foreach (var property in element.EnumerateObject()) + { + foreach (var nested in Walk(property.Value, name)) yield return nested; + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + foreach (var nested in Walk(item, name)) yield return nested; + } + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs new file mode 100644 index 000000000..3bbc96e30 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs @@ -0,0 +1,204 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The fault injector's HTTP surface, as much of it as this suite needs. +/// +/// +/// Deliberately the same shape go-redis, redis-py and node-redis wrap: POST /action returning an id and +/// GET /action/{id} to poll. They integrated independently and converged, so the contract is stable +/// enough to depend on, and their scenario documentation transfers to this client. +/// +public sealed class FaultInjectorClient(Uri baseAddress) : IDisposable +{ + private readonly HttpClient _http = new() { BaseAddress = baseAddress, Timeout = TimeSpan.FromMinutes(2) }; + + /// + /// Statuses that mean "still going". Both of them, which is the point. + /// + /// + /// A job passes through pending *and* running, so a loop that waits only on pending + /// returns while the work is still in flight - and then the test asserts against a cluster that has not + /// finished changing. Documented as a trap in the console's notes; encoded here so it cannot be + /// rediscovered. + /// + private static readonly HashSet PendingStatuses = new(StringComparer.OrdinalIgnoreCase) { "pending", "running", "in_progress" }; + + private static readonly HashSet FailedStatuses = new(StringComparer.OrdinalIgnoreCase) { "failed", "cancelled", "error" }; + + public void Dispose() => _http.Dispose(); + + /// + /// Fires an action and returns its id, without waiting. + /// + public async Task StartActionAsync(string type, object? parameters = null, CancellationToken cancellationToken = default) + { + var payload = new ActionRequest(type, parameters); + using var response = await _http.PostAsJsonAsync("/action", payload, cancellationToken); + await EnsureSuccessAsync(response, $"POST /action ({type})", cancellationToken); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + return ReadActionId(document.RootElement) + ?? throw new InvalidOperationException($"the injector accepted '{type}' but returned no action id: {document.RootElement}"); + } + + /// + /// Fires an action and waits for it to finish, returning its final payload. + /// + public async Task RunActionAsync( + string type, + object? parameters = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var id = await StartActionAsync(type, parameters, cancellationToken); + return await WaitForActionAsync(id, timeout, cancellationToken); + } + + /// + /// Polls an action to completion. + /// + /// + /// The timeout is generous by default because these actions move data: a slot migration or a node coming + /// out of maintenance mode is minutes, not seconds. A tight default here would produce failures that look + /// like product bugs. + /// + public async Task WaitForActionAsync( + string id, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromMinutes(10)); + string? lastStatus = null; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + using var response = await _http.GetAsync($"/action/{id}", cancellationToken); + await EnsureSuccessAsync(response, $"GET /action/{id}", cancellationToken); + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(body); + var status = ReadStatus(document.RootElement); + lastStatus = status ?? lastStatus; + + if (status is not null && !PendingStatuses.Contains(status)) + { + if (FailedStatuses.Contains(status)) + { + throw new InvalidOperationException($"fault-injector action {id} ended as '{status}': {body}"); + } + + // clone: the JsonDocument is disposed with this scope, and callers keep the result + return document.RootElement.Clone(); + } + + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"fault-injector action {id} was still '{lastStatus ?? "unknown"}' after {(timeout ?? TimeSpan.FromMinutes(10)).TotalSeconds:0}s"); + } + + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + } + + /// + /// Discovers which triggers are valid for an effect. + /// + /// + /// Worth asking rather than hardcoding: the effect/trigger matrix is sparse - maintenance_mode only + /// supports remove-add and remove, failover needs replication enabled - and some + /// scenarios do not enumerate their triggers in the schema at all. + /// + public async Task GetValidTriggersAsync(string scenario, string effect, int clusterIndex = 0, CancellationToken cancellationToken = default) + { + using var response = await _http.GetAsync($"/{scenario}?effect={Uri.EscapeDataString(effect)}&cluster_index={clusterIndex}", cancellationToken); + await EnsureSuccessAsync(response, $"GET /{scenario}?effect={effect}", cancellationToken); + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + return document.RootElement.Clone(); + } + + /// + /// Runs a scenario's setup / run / teardown triple's individual legs. + /// + public async Task PostScenarioAsync( + string scenario, + string? leg, + IReadOnlyDictionary query, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var path = leg is null ? $"/{scenario}" : $"/{scenario}/{leg}"; + var separator = '?'; + foreach (var pair in query) + { + if (pair.Value is null) continue; + path += $"{separator}{pair.Key}={Uri.EscapeDataString(pair.Value)}"; + separator = '&'; + } + + using var response = await _http.PostAsync(path, content: null, cancellationToken); + await EnsureSuccessAsync(response, $"POST {path}", cancellationToken); + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(body); + + // the scenario legs only *enqueue*; a caller that returns here is racing the work it asked for + var id = ReadActionId(document.RootElement); + return id is null ? document.RootElement.Clone() : await WaitForActionAsync(id, timeout, cancellationToken); + } + + private static async Task EnsureSuccessAsync(HttpResponseMessage response, string what, CancellationToken cancellationToken) + { + if (response.IsSuccessStatusCode) return; + + // include the body: the injector explains refusals there, and "400 Bad Request" alone has cost people + // hours on the effect/trigger matrix + var body = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"{what} failed: {(int)response.StatusCode} {response.ReasonPhrase}. {body}"); + } + + /// + /// The id of a pollable action, if this response describes one. + /// + /// + /// Deliberately not setup_id: a scenario's setup returns a *handle* to state held in the injector, + /// which is passed back on the run and teardown legs, and is not something /action/{id} knows about. + /// Treating it as an action id polls a URL that does not exist. + /// + private static string? ReadActionId(JsonElement element) + { + foreach (var name in new[] { "action_id", "id" }) + { + if (element.TryGetProperty(name, out var value)) + { + return value.ValueKind switch + { + JsonValueKind.String => value.GetString(), + JsonValueKind.Number => value.ToString(), + _ => null, + }; + } + } + + return null; + } + + private static string? ReadStatus(JsonElement element) + => element.TryGetProperty("status", out var status) && status.ValueKind == JsonValueKind.String + ? status.GetString() + : null; + + private sealed record ActionRequest( + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("parameters")] object? Parameters); +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs new file mode 100644 index 000000000..6c90a2f6b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// One run of a fault-injector scenario: setup, fire, tear down. +/// +/// +/// The setup leg provisions its own database, which is the important part: every trigger publishes the +/// dbconfig it needs (GET /topology-change-standalone?effect=...), and all of them want +/// proxy_policy: single - a shape the environment templates do not create. So a scenario cannot be +/// pointed at an existing database, and there is no need to hand-roll create_database for one either. +/// +/// Teardown is the whole reason this is a type rather than three calls. A scenario left set up holds cluster +/// state - the flags it enabled, the database it made, nodes it excluded - and poisons every run after it. +/// +/// +public sealed class ScenarioRun : IAsyncDisposable +{ + private readonly FaultInjectorClient _injector; + private readonly string _scenario; + private readonly Action _log; + + private ScenarioRun(FaultInjectorClient injector, string scenario, string effect, string trigger, Action log) + { + _injector = injector; + _scenario = scenario; + Effect = effect; + Trigger = trigger; + _log = log; + } + + public string Effect { get; } + + public string Trigger { get; } + + /// The setup handle, passed back on the run and teardown legs. + public string? SetupId { get; private set; } + + /// The database the setup leg created, when it says which. + public int? BdbId { get; private set; } + + /// Whatever setup returned, for a test that wants to look at more than we model. + public JsonElement SetupResult { get; private set; } + + /// The database setup created, ready to connect to. + /// + /// Setup hands back everything needed - endpoint, password, TLS - so a scenario test never has to look in + /// endpoints.json or ask the cluster REST API. Verified against a live run: the response carries + /// setup_id, bdb_id, db_name, endpoints, password, tls, + /// mtls_files and config (the proxy policy it chose to satisfy the trigger). + /// + public ScenarioDatabase? Database { get; private set; } + + /// The database a scenario provisioned for itself. + public sealed record ScenarioDatabase(string Name, int BdbId, string Host, int Port, bool Tls, string? Password, string? ProxyPolicy) + { + public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, policy {ProxyPolicy ?? "?"})"; + + public ConfigurationOptions GetClientConfig( + FaultInjectorEnvironment environment, + MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = mode, + AbortOnConnectFail = false, + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Tls) + { + options.Ssl = true; + options.SslHost = Host; + options.TrustIssuer(environment.CertificateAuthorityPath + ?? throw new InvalidOperationException($"{Name} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}")); + } + + return options; + } + } + + /// + /// Sets a scenario up, ready to fire. + /// + /// + /// The trigger for the *setup* leg, when it differs from the one being fired. + /// + /// + /// The two triggers are different questions, which the naming hides. The setup leg's trigger says how to + /// *provision* - /slot-migrate/setup accepts only reshard, because provisioning a cluster + /// database is all it does - while the run leg's trigger says how to cause the effect (migrate, + /// maintenance_mode, failover). Passing the run trigger to setup earns a + /// "Trigger 'migrate' is not supported by slot-migrate/setup (only 'reshard')". + /// + public static async Task SetupAsync( + FaultInjectorClient injector, + string scenario, + string effect, + string trigger, + Action log, + IReadOnlyDictionary? extra = null, + string? setupTrigger = null, + CancellationToken cancellationToken = default) + { + var run = new ScenarioRun(injector, scenario, effect, trigger, log); + var query = new Dictionary { ["effect"] = effect, ["trigger"] = setupTrigger ?? trigger }; + if (extra is not null) + { + foreach (var pair in extra) query[pair.Key] = pair.Value; + } + + log($"setup {scenario}: effect={effect} setup-trigger={setupTrigger ?? trigger} (firing '{trigger}')"); + run.SetupResult = await injector.PostScenarioAsync(scenario, "setup", query, cancellationToken: cancellationToken); + run.SetupId = FindString(run.SetupResult, "setup_id"); + run.BdbId = FindInt(run.SetupResult, "bdb_id"); + run.Database = ReadDatabase(run.SetupResult, run.BdbId); + log($"setup complete: setup_id={run.SetupId ?? "(none)"} database={run.Database?.ToString() ?? "(none)"}"); + return run; + } + + /// + /// Fires the scenario and waits for the injector to finish its work. + /// + /// + /// Note "finished" here means the injector has done what it was asked, not that the deployment has settled: + /// the notifications, the DNS change and the socket close all trail it by seconds. Callers should watch for + /// what they expect rather than assuming completion means arrival. + /// + public async Task FireAsync(CancellationToken cancellationToken = default) + { + var query = new Dictionary + { + ["effect"] = Effect, + ["trigger"] = Trigger, + ["setup_id"] = SetupId, + ["bdb_id"] = BdbId?.ToString(), + }; + + _log($"firing {_scenario}"); + var result = await _injector.PostScenarioAsync(_scenario, leg: null, query, cancellationToken: cancellationToken); + _log($"fired: {result}"); + return result; + } + + public async ValueTask DisposeAsync() + { + var query = new Dictionary + { + // both, because setup_id lives in the injector's memory and is lost if it restarts, at which point + // bdb_id is the only handle left + ["setup_id"] = SetupId, + ["bdb_id"] = BdbId?.ToString(), + ["restore_nodes"] = "true", // put back anything the scenario excluded, or the next run starts degraded + }; + + try + { + // Its own budget rather than the caller's token: teardown has to happen even when the test was + // cancelled or timed out, which is exactly when the caller's token is already dead. + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await _injector.PostScenarioAsync(_scenario, "teardown", query, cancellationToken: timeout.Token); + _log("teardown complete"); + } + catch (Exception ex) + { + // loud, but not an exception: a teardown failure must not replace the test's own verdict + _log($"TEARDOWN FAILED ({ex.Message}) - the cluster may be left with a scenario set up, and " + + $"setup_id={SetupId ?? "(none)"} bdb_id={BdbId?.ToString() ?? "(none)"} is what to clean up by hand"); + } + } + + private static ScenarioDatabase? ReadDatabase(JsonElement setup, int? bdbId) + { + if (bdbId is not { } id) return null; + + var endpoint = FindString(setup, "endpoints"); + if (endpoint is null) return null; + + var text = endpoint; + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) text = text[(scheme + 3)..]; + var colon = text.LastIndexOf(':'); + if (colon <= 0 || !int.TryParse(text[(colon + 1)..], out var port)) return null; + + return new ScenarioDatabase( + FindString(setup, "db_name") ?? $"bdb-{id}", + id, + text[..colon], + port, + setup.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, + FindString(setup, "password"), + FindString(setup, "config")); + } + + private static string? FindString(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.String) return candidate.GetString(); + if (candidate.ValueKind == JsonValueKind.Number) return candidate.ToString(); + if (candidate.ValueKind == JsonValueKind.Array && candidate.GetArrayLength() > 0 + && candidate[0].ValueKind == JsonValueKind.String) + { + return candidate[0].GetString(); // "endpoints" is a list even when there is one + } + } + + return null; + } + + private static int? FindInt(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.Number && candidate.TryGetInt32(out var value)) return value; + if (candidate.ValueKind == JsonValueKind.String && int.TryParse(candidate.GetString(), out var parsed)) return parsed; + } + + return null; + } + + /// + /// Yields every value for a property name at any depth, shallowest first. + /// + /// + /// Tolerant on purpose: the scenario responses are documented as prose and nest differently between legs, so + /// searching beats asserting a shape - and a test that cannot find setup_id still has a logged + /// payload to work from rather than a deserialization error. + /// + private static IEnumerable Walk(JsonElement element, string name) + { + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty(name, out var direct)) yield return direct; + foreach (var property in element.EnumerateObject()) + { + foreach (var nested in Walk(property.Value, name)) yield return nested; + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + foreach (var nested in Walk(item, name)) yield return nested; + } + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs new file mode 100644 index 000000000..9886ee68b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs @@ -0,0 +1,95 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Telling "this deployment cannot do that" apart from "the client got it wrong". +/// +/// +/// The effect/trigger matrix is sparse in ways that depend on the *cluster*, not just the database: an +/// add migration needs a node holding more than one shard, and a three-node cluster with sparse shard +/// placement gives every node exactly one. The injector reports that as a failed action with a Python +/// traceback, which is indistinguishable from a real fault unless you read the message. +/// +/// So these skip, narrowly and by message. A broad catch here would hide genuine injector failures, which is +/// the thing this tier exists to surface - hence matching on the specific condition rather than on the +/// exception type. +/// +/// +internal static class ScenarioSupport +{ + private static readonly string[] PlacementLimitations = + [ + "No node with multiple shards found", + "not enough nodes", + "no empty node", + ]; + + /// + /// Skips when the setup could not produce a database the effect can act on. + /// + public static void RequireEffectIsAchievable(ScenarioRun scenario, string effect) + { + if (scenario.Database is null) + { + Assert.Skip($"the injector did not provision a database for '{effect}'"); + } + } + + /// + /// Fires a scenario, skipping rather than failing when the cluster's shape cannot produce the effect. + /// + public static async Task FireOrSkipAsync(ScenarioRun scenario, string effect, CancellationToken cancellationToken) + { + try + { + await scenario.FireAsync(cancellationToken); + } + catch (Exception ex) when (IsPlacementLimitation(ex)) + { + Assert.Skip($"this cluster cannot produce '{effect}': {Summarize(ex.Message)}"); + } + } + + private static bool IsPlacementLimitation(Exception ex) + { + foreach (var limitation in PlacementLimitations) + { + if (ex.Message.Contains(limitation, StringComparison.OrdinalIgnoreCase)) return true; + } + + return false; + } + + /// + /// The part of a Python traceback that says what actually went wrong. + /// + /// + /// Note the traceback arrives inside a JSON string, so its line breaks are the two characters + /// \n rather than real newlines - splitting on '\n' alone matches nothing and returns + /// the whole wall of text, which is how the first version of this behaved. The injector helpfully ends with + /// "Caused by: ...", which is the only part worth putting in a skip message. + /// + internal static string Summarize(string message) + { + var caused = message.LastIndexOf("Caused by:", StringComparison.Ordinal); + if (caused >= 0) + { + var tail = message[caused..]; + var end = tail.IndexOf("\\n", StringComparison.Ordinal); + return end > 0 ? tail[..end] : tail; + } + + var lines = message.Replace("\\n", "\n", StringComparison.Ordinal) + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + for (int i = lines.Length - 1; i >= 0; i--) + { + if (lines[i].Contains("Exception:", StringComparison.Ordinal)) return lines[i]; + } + + return lines.Length > 0 ? lines[^1] : message; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs new file mode 100644 index 000000000..14fe17dbd --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// D6 against a real deployment: do we get off the connection before the server takes it away? +/// +/// +/// The one thing the in-process harness structurally cannot test. The handoff's hostname branch needs two +/// things a fake cannot supply - an endpoint that is a *name*, and a real socket whose remote address tells us +/// where we currently are - and its decision turns on DNS changing underneath us. +/// +/// The discriminator is the failure type of the disconnect. ConnectionDisposed means *we* replaced the +/// connection; SocketClosed means the server did. Both outcomes are legitimate, because DNS is not +/// guaranteed to win the race - measured on one cluster at +18.7s against a socket closing at +15.7s - so this +/// records which happened rather than demanding the good one. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "moving-handoff")] +public class MovingHandoffScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("conn_drop", "endpoint_rebind")] + [InlineData("data_movement_conn_drop", "maintenance_mode")] + public async Task HandoffBeatsTheServerToTheClose(string effect, string trigger) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", effect, trigger, log.WriteLine, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var clock = Stopwatch.StartNew(); + var timeline = new List(); + void Note(string what) + { + var entry = $" +{clock.Elapsed.TotalSeconds,6:0.0}s {what}"; + lock (timeline) timeline.Add(entry); + log.WriteLine(entry); + } + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var muxer = (IInternalConnectionMultiplexer)conn; + var endpoint = muxer.GetServerEndPoint(conn.GetEndPoints()[0]); + + // The precondition the fake could not meet: a name to resolve, and a socket that says where we are. + Assert.IsType(conn.GetEndPoints()[0]); + log.WriteLine($"dialled {conn.GetEndPoints()[0]} (a name, so the probe can engage)"); + + var failures = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (failures) failures.Add(e.FailureType); + Note($"disconnect: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => Note("reconnected"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) Note($"{push.NotificationType} seq={push.SequenceId} time={push.Time?.TotalSeconds.ToString() ?? "-"}"); + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + Note("injector reports the scenario finished"); + + // long enough to cover the whole announced window plus the observed overshoot + var deadline = clock.Elapsed + TimeSpan.FromSeconds(75); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + Note($"ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + Note($"handoff outcome: {endpoint.LastHandoffOutcome ?? "(none recorded)"}; recycles={endpoint.HandoffRecycles}"); + + lock (failures) + { + var order = string.Join(" -> ", failures); + log.WriteLine($" disconnect sequence: {(order.Length == 0 ? "(none)" : order)}"); + + // Note what is *not* asserted: that a ConnectionDisposed appears here. Our own recycle does not + // raise ConnectionFailed - disposal is not reported as a failure - so from the outside a handoff is + // invisible, which is worth knowing in its own right and is why HandoffRecycles exists. + } + + // Exactly one handoff per notification. More than one is the feedback loop this test found on its first + // live run: a server re-sends MOVING to a connection that opts in while the window is still open, and + // since the handoff replaces the connection, acting on the repeat produces another one - twelve + // recycles from a single event. The per-server sequence dedup now gates it, and this is the assertion + // that would catch a regression. + Assert.Equal(1, endpoint.HandoffRecycles); + Assert.NotNull(endpoint.LastHandoffOutcome); + Assert.Contains("Recycle", endpoint.LastHandoffOutcome); + + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should be serving commands after the handoff"); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs new file mode 100644 index 000000000..0d93b2db2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs @@ -0,0 +1,21 @@ +using System; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Waits for a condition that a real deployment reaches on its own schedule. +/// +internal static class Poll +{ + public static async Task UntilAsync(Func condition, int timeoutMilliseconds = 10_000, int pollMilliseconds = 250) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMilliseconds); + while (true) + { + if (condition()) return true; + if (DateTime.UtcNow > deadline) return false; + await Task.Delay(pollMilliseconds); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs new file mode 100644 index 000000000..dd2135c74 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A replicated database, provisioned by us - the only way to reach the failover family. +/// +/// +/// The environment templates create databases with replication: false, and a failover needs a replica to +/// promote, so this is the one case where create_database earns its keep rather than being a +/// less-convenient alternative to the scenario setup legs. +/// +public sealed class ReplicatedDatabaseFixture() + : FaultInjectorFixture(new DatabaseShape("replicated", ProxyPolicy: "single", Replication: true, ShardCount: 2)); + +/// +/// The proxy and failover families: FAILING_OVER/FAILED_OVER, and a proxy restarting underneath us. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "failover")] +public class FailoverScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Fact] + public async Task FailoverIsAnnouncedAndSurvived() + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + log.WriteLine($"provisioned {database}"); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + } + }; + + var db = conn.GetDatabase(); + await db.StringSetAsync("fi-failover", "before"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "failover", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + // The failover action's parameters are untyped in the schema, so a rejection here is a harness + // problem rather than a client finding; say so plainly instead of reporting it as a product failure. + Assert.Skip($"the injector would not run 'failover' against bdb {database.BdbId}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports the failover finished"); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + try + { + await db.PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s): {string.Join(", ", events.Select(e => e.NotificationType))}"); + + // A failover is the one event whose *pair* we have never observed end to end from a client: Marc + // captured the frames by hand, but nothing has watched SE.Redis receive them. + Assert.NotEmpty(events); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.FailingOver + or MaintenanceNotificationType.FailedOver + or MaintenanceNotificationType.Migrating // a failover on a proxied database moves shards too + or MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.Moving); + } + + // the data survived, which is the point of replication + Assert.Equal("before", await db.StringGetAsync("fi-failover")); + } +} + +/// +/// The proxy process restarting: no topology change, no notification, just the socket going away. +/// +/// +/// Included because it is the one disruption that is *purely* a connection event - nothing moves, nothing is +/// announced, and recovery is entirely the client's ordinary reconnect path. A useful control: if this fails, +/// the failures in the announced scenarios are not about notifications at all. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "proxy-restart")] +public class ProxyRestartScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("standalone")] + [InlineData("cluster")] + public async Task ProxyRestartIsSurvived(string key) + { + var database = fixture.Require(key); + var cancellationToken = TestContext.Current.CancellationToken; + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var db = conn.GetDatabase(); + await db.StringSetAsync($"fi-dmc-{key}", "before"); + + int drops = 0; + conn.ConnectionFailed += (_, e) => + { + drops++; + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s connection failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s connection restored"); + + clock.Restart(); + // the one action with a typed parameter object in the injector's schema: RestartDmcParams { bdb_id } + await fixture.Injector.RunActionAsync( + "dmc_restart", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s DMC restart reported finished"); + + // Recovery is asserted by polling rather than by waiting a fixed time, because a proxy restart is quick + // and the interesting failure is "never comes back", not "takes a moment". + Assert.True( + await Poll.UntilAsync(() => TryRead(db, $"fi-dmc-{key}"), timeoutMilliseconds: 60_000), + "the client should recover from the proxy restarting"); + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered after {drops} drop(s)"); + Assert.Equal("before", await db.StringGetAsync($"fi-dmc-{key}")); + } + + private static bool TryRead(IDatabase db, string key) + { + try + { + return db.StringGet(key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md new file mode 100644 index 000000000..2be42b3a7 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -0,0 +1,60 @@ +# StackExchange.Redis.FaultInjector.Tests + +Scenario tests that drive a **real Redis Enterprise deployment** through the fault injector, and watch how +SE.Redis reacts. These are the tier that can observe what no in-process fake can: real DNS, real TLS identity, +real timing. + +## Running them + +```bash +cd # holds docker-compose.yml, env_output.json, ... +docker compose up -d # 10-15 minutes for the cluster to come up + +export SER_FI_CONFIG_DIR=$PWD # or FI_CONSOLE_CONFIG_DIR, which is also honoured +export E2E_SCENARIO_TESTS=true # explicit opt-in: these create and delete databases +dotnet test tests/StackExchange.Redis.FaultInjector.Tests +``` + +One path is the whole configuration. That directory is the one mounted into the injector as `/app/config`, so +it already holds the cluster credentials (`env_output.json`), the CA certificate, and the compose file; nothing +has to be hand-carried into the test run. `FAULT_INJECTION_API_URL` overrides the injector URL +(default `http://127.0.0.1:20324`). + +## Three states, deliberately distinct + +| state | behaviour | +|---|---| +| no `SER_FI_CONFIG_DIR` | every test **skips** - the ordinary case, including `build.ps1`'s full traversal | +| directory set, no `E2E_SCENARIO_TESTS=true` | every test **skips** - nobody should create databases by accident | +| configured and enabled, but broken | every test **fails** | + +The third row is the important one. A suite that skips when the environment is broken reports success for tests +that never ran, and it will be trusted at exactly the wrong moment. + +## Databases are created by the tests, not by you + +Each *shape* (`DatabaseShape`) is a fixture shared by the classes that need it, because creating a database on a +real cluster is slow. The shapes exist because they change client behaviour rather than for coverage's sake: the +number of A records a hostname carries follows proxy placement, and the handoff takes a different branch +depending on whether a live sibling address exists. + +Every database is named `sertest--`. Cleanup is per fixture and unconditional; a sweep at startup +removes leaks from runs that were killed, matching on the `sertest-` prefix and nothing else, so it can never +touch a database created by hand. + +## TLS + +Certificates are self-signed per environment, so tests call `ConfigurationOptions.TrustIssuer(caPath)` with the +CA found in the config directory. If the CA is missing, TLS tests **fail** rather than disabling validation - a +TLS test that quietly stops checking identity reports success for the one thing it exists to catch. + +## Traits + +`tier=fault-injector` on everything, so the whole tier can be excluded in one filter; `scenario=` for +subsets. + +## Unverified + +The `create_database` parameter names in `DatabaseShape.ToCreateParameters` are the injector's wire schema, +which is documented only as prose. They are gathered in one place so a real run can correct them; go-redis's +`DatabaseConfig` is the closest reference implementation. Treat them as unconfirmed until a run accepts them. diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs new file mode 100644 index 000000000..a5f37b8b6 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Does the client work against a real deployment at all: connect, negotiate RESP3, opt in, run a command. +/// +/// +/// The floor for everything else. Worth having as its own class because when a scenario test fails, the first +/// question is whether the deployment was reachable in the first place, and a green smoke test answers it +/// without reading logs. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "smoke")] +public class RealDeploymentSmokeTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("standalone")] + [InlineData("cluster")] + public async Task OptInIsAcceptedAndTheConnectionWorks(string key) + { + var database = fixture.Require(key); + log.WriteLine($"{database}"); + log.WriteLine($" advertised addresses: {string.Join(", ", database.Addresses)}"); + log.WriteLine($" endpoint type: {database.EndpointType ?? "(unset)"}"); + + // Enabled refuses the connection if the server will not give us notifications, so reaching the + // assertions at all is the opt-in having been accepted - there is nothing weaker to check. A stub +OK + // would pass this, which is what the scenario tests are for. + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + Assert.True(conn.IsConnected); + + var server = conn.GetServer(conn.GetEndPoints()[0]); + log.WriteLine($" connected: {server.Version}, {server.ServerType}, protocol {server.Protocol}"); + Assert.Equal(RedisProtocol.Resp3, server.Protocol); + + var rtt = await conn.GetDatabase().PingAsync(); + log.WriteLine($" ping: {rtt.TotalMilliseconds:0.0}ms"); + Assert.True(rtt > TimeSpan.Zero); + + // and the feature is actually live on this connection, not merely requested + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "maintenance notifications should be active"); + } + + [Fact] + public async Task AdvertisedAddressCountMatchesTheProxyPolicy() + { + // Not a client assertion: a check that the environment is the shape the handoff tests assume. The count + // follows proxy *placement* rather than the policy name, so this records what this deployment actually + // is - and a handoff test that expects a sibling to step to needs more than one. + var standalone = fixture.Require("standalone"); + log.WriteLine($"{standalone.Key}: policy {standalone.ProxyPolicy}, {standalone.AdvertisedAddressCount} address(es)"); + + Assert.NotEmpty(standalone.Addresses); + if (standalone.ProxyPolicy is "single") + { + Assert.Equal(1, standalone.AdvertisedAddressCount); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj new file mode 100644 index 000000000..052d85e11 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + Exe + StackExchange.Redis.FaultInjector.Tests + true + true + full + enable + true + + + + + + + + + + + + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs new file mode 100644 index 000000000..a99be3bc2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The same feature over TLS, against certificates a real cluster generated. +/// +/// +/// A gap that has been open since the identity work: nothing had exercised a real TLS deployment, only the +/// in-process harness with a certificate it made itself. That matters because the interesting failures are +/// about *identity* - which name the certificate carries, and whether the endpoint we are told to use is +/// covered by it - and a fake that issues its own certificate cannot be wrong about that in the way a real +/// deployment can. +/// +/// Certificates here are self-signed per environment, so the client pins the issuer with +/// . Note what is deliberately *not* done: validation is +/// never disabled. A TLS test that stops checking identity reports success for precisely the thing it exists to +/// catch, so a missing CA fails the run instead. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "tls")] +public class TlsScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Fact] + public async Task NotificationsArriveOverTlsAndIdentityIsVerified() + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + if (fixture.Environment.CertificateAuthorityPath is null) + { + // a skip rather than a failure: whether the environment generated certificates is a provisioning + // choice, and running this without them would prove nothing + Assert.Skip($"no CA certificate in {fixture.Environment.ConfigDirectory.FullName}; provision with TLS to run this"); + } + + log.WriteLine($"trusting issuer {fixture.Environment.CertificateAuthorityPath}"); + CertificateSanity.RequireCertificatesMatchThisCluster(fixture.Environment, log.WriteLine); + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, + "topology-change-standalone", + "conn_drop", + "endpoint_rebind", + log.WriteLine, + // include_tls does not *request* TLS, it widens the list of variants the setup may choose from; + // variant_index is what picks one. Confirmed by asking the discovery endpoint: with no flags a + // trigger offers one variant ("single"), with include_tls two ("single", "single_tls"), and with + // include_mtls a third ("mtls"). Passing include_tls alone provisions variant 0 and yields a + // plaintext database, which is how this test first came to skip itself. + extra: new Dictionary + { + ["include_tls"] = "true", + ["variant_index"] = "1", + }, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + log.WriteLine($"provisioned {database} (tls={database.Tls})"); + + if (!database.Tls) + { + Assert.Skip("the injector provisioned a plaintext database despite include_tls=true; nothing to test here"); + } + + var clock = Stopwatch.StartNew(); + var events = new List(); + + var config = database.GetClientConfig(fixture.Environment); + + // AbortOnConnectFail=true *for this test only*: everywhere else tolerating a slow start is right, but a + // TLS failure has to surface its reason. With it false, a certificate problem is indistinguishable from + // a slow cluster - ConnectAsync succeeds, IsConnected is false, and the cause is gone. + config.AbortOnConnectFail = true; + + // If the certificate does not cover the name we dialled, this throws - which is the point: the + // assertion that identity was verified is the connect succeeding with validation on. The log goes to + // test output so a handshake failure says *which* check failed. + var connectLog = new StringWriter(); + ConnectionMultiplexer conn; + try + { + conn = await ConnectionMultiplexer.ConnectAsync(config, connectLog); + } + catch (Exception ex) + { + log.WriteLine(connectLog.ToString()); + log.WriteLine($"TLS connect failed: {ex.GetType().Name}: {ex.Message}"); + throw; + } + + await using (conn) + { + Assert.True(conn.IsConnected); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "the opt-in should be live over TLS too"); + log.WriteLine($"connected over TLS; opt-in active; ping {(await conn.GetDatabase().PingAsync()).TotalMilliseconds:0.0}ms"); + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + } + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s) over TLS"); + Assert.NotEmpty(events); + } + + // and the TLS handshake succeeds again on the *replacement* connection, which is the part a + // certificate-name problem would break rather than the first connect + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should re-establish TLS after the endpoint moves"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs new file mode 100644 index 000000000..4c79f1e01 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Real topology changes on a real deployment, watched by a real client. +/// +/// +/// Each scenario provisions its own database, because every trigger publishes the dbconfig it requires +/// and all of them want proxy_policy: single - a shape the environment templates do not create. +/// +/// What these assert is deliberately modest: that the notifications arrive, parse, and open the relaxation +/// window. Timings are *recorded* rather than asserted, because the measured spread across clusters is wide +/// enough that a bound tight enough to be interesting would be flaky - DNS has been seen updating 4.4s after +/// MOVING on one cluster and 3s after the socket closed on another. The log is the deliverable for +/// timing; the assertions cover behaviour. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "topology-change")] +public class TopologyChangeScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + /// + /// Records what arrived and when, relative to the moment the scenario was fired. + /// + private sealed class Timeline(Stopwatch clock, ITestOutputHelper log) + { + private readonly List _entries = []; + + public List Events { get; } = []; + + public void Note(string what) + { + var entry = $" +{clock.Elapsed.TotalSeconds,6:0.0}s {what}"; + lock (_entries) + { + _entries.Add(entry); + } + + log.WriteLine(entry); + } + + public void Add(PushMaintenanceEvent evt) + { + lock (_entries) + { + Events.Add(evt); + } + + Note($"{evt.NotificationType} seq={evt.SequenceId} time={evt.Time?.TotalSeconds.ToString() ?? "-"} {evt.RawMessage}"); + } + + public int Count + { + get { lock (_entries) { return Events.Count; } } + } + } + + /// + /// Whether each scenario announces itself, and why - measured 2026-09-01 against RS 8.0.22. + /// + /// + /// The expectations encode the rule the measurements produced, which is narrower than either half of it + /// looks: a connection is told MOVING when its own proxy leaves the endpoint's address set *and* + /// the set gains a member. Both conditions are needed - a pure reduction takes the proxy away without + /// announcing it, and a pure widening adds addresses while leaving the connection's proxy in place, which is + /// also silent. The data-movement pair (MIGRATING/MIGRATED) is separate and fires whenever + /// shards move, whether or not any endpoint changes. + /// + [Theory] + [InlineData("conn_drop", "endpoint_rebind", true)] + [InlineData("dns_resolution_change", "endpoint_rebind", false)] + [InlineData("data_movement_conn_drop", "maintenance_mode", true)] + [InlineData("data_movement_no_conn_drop", "migrate", true)] + public async Task ScenarioProducesNotificationsWeUnderstand(string effect, string trigger, bool expectNotifications) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", effect, trigger, log.WriteLine, cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var clock = Stopwatch.StartNew(); + var timeline = new Timeline(clock, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "the opt-in must be live, or this test proves nothing"); + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) timeline.Add(push); + }; + conn.ConnectionFailed += (_, e) => timeline.Note($"connection failed: {e.FailureType} {e.Exception?.Message}"); + conn.ConnectionRestored += (_, e) => timeline.Note("connection restored"); + + clock.Restart(); + timeline.Note($"firing {effect}/{trigger} against {database}"); + await scenario.FireAsync(cancellationToken); + timeline.Note("injector reports the scenario finished"); + + // The injector finishing is not the deployment settling: notifications, the DNS change and the socket + // close all trail it. Keep watching, and keep the client busy so a broken connection actually surfaces + // rather than sitting idle. + var deadline = clock.Elapsed + TimeSpan.FromSeconds(60); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + timeline.Note($"ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + timeline.Note($"finished with {timeline.Count} notification(s); relaxed={endpoint.IsMaintenanceRelaxed}"); + + if (expectNotifications) + { + Assert.NotEmpty(timeline.Events); + Assert.All(timeline.Events, e => Assert.NotEqual(MaintenanceNotificationType.None, e.NotificationType)); + } + else + { + // dns_resolution_change widens the policy (single -> all-master-shards), so addresses are *added* + // and the connection's own proxy stays where it is - there is nothing to tell this connection to + // move, and the proxy restarting to apply the change closes the socket with no warning at all. + // Asserting the silence deliberately: it is the case with no signal, so if a future build starts + // announcing it, that is a behaviour change we want to be told about rather than to absorb quietly. + Assert.Empty(timeline.Events); + } + + // and the client is usable afterwards, which is the point of the whole feature + Assert.True( + await Poll.UntilAsync(() => TryPing(conn), timeoutMilliseconds: 30_000), + "the client should be serving commands again after the topology change"); + } + + private static bool TryPing(IConnectionMultiplexer conn) + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + } +} diff --git a/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs b/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs new file mode 100644 index 000000000..302c880f1 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs @@ -0,0 +1,216 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The DNS probe behind the MOVING handoff. Tested against an injected resolver rather than real DNS, +/// which is the only way to exercise the case that actually happens: the first answer naming the address we +/// were just told to leave. +/// +public class AdvertisedAddressProbeTests(ITestOutputHelper log) +{ + private static readonly IPAddress Retiring = IPAddress.Parse("10.129.228.140"); + private static readonly IPAddress Replacement = IPAddress.Parse("10.252.90.18"); + private static readonly DnsEndPoint Endpoint = new("db.example.cloud.redislabs.com", 13486); + + /// + /// Resolves from a script: one entry per call, the last repeating forever. + /// + private sealed class ScriptedResolver(params IPAddress[][] answers) + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + + public Task ResolveAsync(string host, CancellationToken cancellationToken) + { + var index = Interlocked.Increment(ref _calls) - 1; + return Task.FromResult(answers[Math.Min(index, answers.Length - 1)]); + } + } + + [Fact] + public async Task DnsTrailingTheNotificationIsPolledThrough() + { + // The measured case: DNS named the retiring address for the first 4.4-9.7 seconds after MOVING. A + // client that accepted the first answer would hand off to the node it was told to leave. + var resolver = new ScriptedResolver( + [Retiring], + [Retiring], + [Retiring], + [Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(4, resolver.Calls); + } + + [Fact] + public async Task ReplacementOnTheFirstAnswerIsTakenImmediately() + { + var resolver = new ScriptedResolver([Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Fact] + public async Task WindowExpiringWithoutAMoveGivesUpRatherThanGuessing() + { + // Not a failure to report: the server closes the socket regardless, and the relaxed window covers the + // reconnect. Guessing an address here would be worse than doing nothing. + var resolver = new ScriptedResolver([Retiring]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromMilliseconds(120), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Null(result); + Assert.True(resolver.Calls > 1, $"should have retried within the window, but resolved {resolver.Calls} time(s)"); + } + + [Fact] + public async Task ZeroWindowStillGetsOneAttempt() + { + // "act now" rather than "do nothing": the notifications legitimately carry zero or negative times for + // a connection that arrived mid-window + var resolver = new ScriptedResolver([Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.Zero, pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Fact] + public async Task ResolutionFailureIsRetriedNotFatal() + { + // a DNS blip mid-handoff is when we can least afford to give up + int calls = 0; + Task Resolve(string host, CancellationToken cancellationToken) + { + calls++; + return calls switch + { + 1 => throw new System.Net.Sockets.SocketException(11001), // host not found + 2 => Task.FromResult([Retiring]), + _ => Task.FromResult([Replacement]), + }; + } + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolve, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(3, calls); + } + + [Fact] + public async Task MultipleAddressesTakeTheOneThatIsNotRetiring() + { + // a round-robin record can name both nodes at once mid-move + var resolver = new ScriptedResolver([Retiring, Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + } + + [Fact] + public async Task PlacementNotPolicyDecidesWhetherWeWait() + { + // The A-record count follows actual proxy placement, not the policy name: an all-master-shards + // database whose shards share a node resolves to one address, and then there is no sibling to step to + // and the wait is the only option. Same code path as `single`, which is the point - nothing here reads + // the policy. + var resolver = new ScriptedResolver([Retiring], [Retiring], [Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(3, resolver.Calls); // it waited, because there was nothing else advertised + } + + [Fact] + public async Task SiblingIsTakenWithoutWaitingForTheRecordToMove() + { + // The common case: several A records, so the first resolution already names a live sibling proxy while + // the retiring address is *still* advertised. Stepping sideways immediately is correct - any proxy of + // the same database serves the same data - and it means the poll usually never engages. + var sibling = IPAddress.Parse("10.246.250.155"); + var resolver = new ScriptedResolver([Retiring, sibling]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(sibling, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task StillAdvertisedAnswersTheUnannouncedCase(bool present) + { + // The measured gap: a multi-proxy node taken out for maintenance announced only MIGRATING/MIGRATED, + // dropped the victim from DNS at +21.4s, and closed its socket silently at +34.7s. For thirteen + // seconds the condition was visible to anyone who asked, and no notification said so. + var sibling = IPAddress.Parse("10.246.250.155"); + var resolver = new ScriptedResolver(present ? [Retiring, sibling] : [sibling, Replacement]); + + var result = await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(present, result); + } + + [Fact] + public async Task ResolutionFailureIsNotAReasonToAbandonAConnection() + { + // null rather than false, deliberately: "cannot tell" must not become "give it up", or a DNS blip + // recycles every healthy connection at once + Task Throws(string host, CancellationToken cancellationToken) + => throw new System.Net.Sockets.SocketException(11001); + + Assert.Null(await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: Throws, log: log.WriteLine)); + + // and the same for a record that momentarily resolves to nothing at all + Assert.Null(await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: (_, _) => Task.FromResult([]), log: log.WriteLine)); + } + + [Fact] + public async Task CancellationStopsThePoll() + { + using var cts = new CancellationTokenSource(); + var resolver = new ScriptedResolver([Retiring]); + + var probe = AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromMinutes(1), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine, cancellationToken: cts.Token); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => probe); + } +} diff --git a/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs new file mode 100644 index 000000000..13e551c1f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs @@ -0,0 +1,61 @@ +using System.Linq; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The comma-and-range slot form the cluster notifications carry, e.g. 123,456,789-1000. Split out +/// because it is the one part of stage 3 that a wire capture cannot invalidate - it is the same form +/// CLUSTER NODES has always used. +/// +public class ClusterSlotMigrationUnitTests(ITestOutputHelper log) +{ + [Theory] + [InlineData("123", "123-123")] + [InlineData("123,456", "123-123,456-456")] + [InlineData("789-1000", "789-1000")] + [InlineData("123,456,789-1000", "123-123,456-456,789-1000")] + [InlineData("0-16383", "0-16383")] + [InlineData("5-5", "5-5")] + public void ParsesTheSlotForm(string input, string expected) + { + Assert.True(SlotRange.TryParseList(input, out var ranges)); + var actual = string.Join(",", ranges.Select(x => $"{x.From}-{x.To}")); + log.WriteLine($"{input} -> {actual}"); + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("123,,456", "123-123,456-456")] // empty element, skipped + [InlineData("123,", "123-123")] // trailing comma + [InlineData(",123", "123-123")] // leading comma + public void ToleratesEmptyElements(string input, string expected) + { + Assert.True(SlotRange.TryParseList(input, out var ranges)); + Assert.Equal(expected, string.Join(",", ranges.Select(x => $"{x.From}-{x.To}"))); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(",")] + [InlineData("abc")] + [InlineData("12a")] + [InlineData("-5")] // no lower bound + [InlineData("5-")] // no upper bound + [InlineData("1000-789")] // reversed: a server bug, not something to normalize silently + [InlineData("1-2-3")] + [InlineData("123,abc")] // one bad element fails the list: a partial slot set is worse than none + public void RejectsMalformedInput(string? input) + { + Assert.False(SlotRange.TryParseList(input, out var ranges)); + log.WriteLine($"'{input}' rejected, {ranges.Count} range(s)"); + } + + [Fact] + public void OutOfRangeSlotIsRejected() + { + // 16384 does not fit the short, and a checked cast would throw rather than wrap + Assert.False(SlotRange.TryParseList("99999", out _)); + } +} diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 6c62e3bb2..06876e946 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -62,6 +62,10 @@ orderby name Assert.Equal( new[] { + "_maintenanceNotifications", + "_maintenancePostEventRelaxedDuration", + "_maintenanceRelaxedTimeout", + "_maintenanceRelaxedWindowMax", "_protocol", "asyncTimeout", "backlogPolicy", @@ -891,6 +895,68 @@ public void CheckHighIntegrity(bool? assigned, bool expected, string cs) Assert.Equal(expected, parsed.HighIntegrity); } + [Theory] + [InlineData("maintRelaxedTimeout=20", 20, 60, 40)] + [InlineData("maintRelaxedWindowMax=45", 10, 45, 20)] + [InlineData("maintPostEventRelaxed=0", 10, 30, 0)] + public void MaintenanceDurationsRoundTrip(string cs, int relaxedSeconds, int capSeconds, int tailSeconds) + { + // these are in *seconds*, unlike every other timeout here, because that is the unit the cross-client + // contract names for maintRelaxedTimeout - so a documented value can be pasted between clients + var options = Parse("dummy," + cs); + Assert.Equal(TimeSpan.FromSeconds(relaxedSeconds), options.MaintenanceRelaxedTimeout); + Assert.Equal(TimeSpan.FromSeconds(capSeconds), options.MaintenanceRelaxedWindowMax); + Assert.Equal(TimeSpan.FromSeconds(tailSeconds), options.MaintenancePostEventRelaxedDuration); + + // only the explicitly-set key is serialized; the others stay defaulted + Assert.Equal("dummy," + cs, RemoveTestDefaults(options.ToString())); + + var clone = options.Clone(); + Assert.Equal(options.MaintenanceRelaxedTimeout, clone.MaintenanceRelaxedTimeout); + Assert.Equal("dummy," + cs, RemoveTestDefaults(clone.ToString())); + } + + [Fact] + public void MaintenanceDurationsDefaultRelativeToTheRelaxedTimeout() + { + // the cap and tail are multiples so that raising the relaxed timeout cannot produce a cap below it + var options = Parse("dummy,maintRelaxedTimeout=60"); + Assert.Equal(TimeSpan.FromSeconds(180), options.MaintenanceRelaxedWindowMax); + Assert.Equal(TimeSpan.FromSeconds(120), options.MaintenancePostEventRelaxedDuration); + } + + [Fact] + public void MaintenanceDurationInMillisecondsIsDiagnosed() + { + // the silent-misconfiguration case: somebody assumes milliseconds like every other timeout here and + // writes 30000, which as seconds would be an eight-hour relaxed timeout. The message names the unit + var ex = Assert.Throws(() => Parse("dummy,maintRelaxedTimeout=30000")); + Output.WriteLine(ex.Message); + Assert.Contains("seconds", ex.Message); + Assert.Contains("not in milliseconds", ex.Message); + } + + [Theory] + [InlineData(null, MaintenanceNotificationMode.Disabled, "dummy")] + [InlineData(MaintenanceNotificationMode.Disabled, MaintenanceNotificationMode.Disabled, "dummy,maintNotifications=Disabled")] + [InlineData(MaintenanceNotificationMode.Enabled, MaintenanceNotificationMode.Enabled, "dummy,maintNotifications=Enabled")] + [InlineData(MaintenanceNotificationMode.Auto, MaintenanceNotificationMode.Auto, "dummy,maintNotifications=Auto")] + public void CheckMaintenanceNotifications(MaintenanceNotificationMode? assigned, MaintenanceNotificationMode expected, string cs) + { + var options = Parse("dummy"); + if (assigned.HasValue) options.MaintenanceNotifications = assigned.Value; + + Assert.Equal(expected, options.MaintenanceNotifications); + Assert.Equal(cs, RemoveTestDefaults(options.ToString())); + + var clone = options.Clone(); + Assert.Equal(expected, clone.MaintenanceNotifications); + Assert.Equal(cs, RemoveTestDefaults(clone.ToString())); + + var parsed = Parse(cs); + Assert.Equal(expected, parsed.MaintenanceNotifications); + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -902,6 +968,12 @@ public void DefaultsProviderProtocolNotSerialized(bool clone) if (clone) options = options.Clone(); Assert.Equal(RedisProtocol.Resp3, options.Protocol); Assert.Same(provider, options.Defaults); - Assert.Equal("", options.ToString()); + + // the *values* a provider supplies are still never serialized - that is what this test is for... + Assert.DoesNotContain("protocol=", options.ToString()); + + // ...but the explicit choice of provider now is, by name, so the string describes the configuration it + // was given rather than quietly dropping part of it + Assert.Equal("defaults=amr", options.ToString()); } } diff --git a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs index fae837bd9..7da6ab839 100644 --- a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs +++ b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs @@ -86,6 +86,135 @@ public void IsMatchOnAzureManagedRedisDomain(string hostName) Assert.IsType(provider); } + [Theory] + [InlineData("amr", typeof(AzureManagedRedisOptionsProvider))] + [InlineData("AMR", typeof(AzureManagedRedisOptionsProvider))] // names are case-insensitive, like every other key + [InlineData("azure", typeof(AzureOptionsProvider))] + [InlineData("rediscloud", typeof(RedisCloudOptionsProvider))] + [InlineData("enterprise", typeof(RedisEnterpriseOptionsProvider))] + public void DefaultsProviderCanBeNamedInAConfigurationString(string name, Type expected) + { + // the on-premise case is why this exists: an Enterprise cluster has whatever DNS its operator gave it, + // so IsMatch can never recognize it, and until now the only way to select a provider was to write code + var options = ConfigurationOptions.Parse($"localhost,defaults={name}"); + Assert.IsType(expected, options.Defaults); + + // and it round-trips, because it was chosen rather than inferred + var text = options.ToString(); + Output.WriteLine(text); + Assert.Contains($"defaults={name.ToLowerInvariant()}", text); + Assert.IsType(expected, ConfigurationOptions.Parse(text).Defaults); + } + + [Fact] + public void ProviderToStringPrefersTheName() + { + // for logs: "amr" rather than a namespace-qualified type name + Assert.Equal("amr", new AzureManagedRedisOptionsProvider().ToString()); + Assert.Equal("enterprise", new RedisEnterpriseOptionsProvider().ToString()); + + // ...and an unnameable one still says something useful, which is exactly why serialization tests + // Name rather than ToString: this is never null + var custom = new TestOptionsProvider(".custom"); + Assert.Null(custom.Name); + Assert.Contains(nameof(TestOptionsProvider), custom.ToString()); + } + + [Fact] + public void UnknownDefaultsProviderNameIsRejectedWithTheAlternatives() + { + var ex = Assert.Throws(() => ConfigurationOptions.Parse("localhost,defaults=sideways")); + Output.WriteLine(ex.Message); + Assert.Contains("enterprise", ex.Message); // the message lists what it would have accepted + Assert.Contains("rediscloud", ex.Message); + } + + [Fact] + public void AnInferredDefaultsProviderIsNotSerialized() + { + // the trap this guards: the Defaults getter memoizes an inferred provider into the same field an + // explicit set writes, so without tracking *how* it got there, merely reading the property would make + // an endpoint-derived guess look like a decision - and re-parsing the string would then pin it + var options = ConfigurationOptions.Parse("contoso.cloud.redislabs.com"); + Assert.IsType(options.Defaults); // inferred, and memoized by that read + + Assert.DoesNotContain("defaults=", options.ToString()); + Assert.DoesNotContain("defaults=", options.Clone().ToString()); + } + + [Fact] + public void AnExplicitDefaultsProviderSurvivesCloning() + { + var options = ConfigurationOptions.Parse("localhost,defaults=enterprise"); + var clone = options.Clone(); + + Assert.IsType(clone.Defaults); + Assert.Contains("defaults=enterprise", clone.ToString()); + } + + [Fact] + public void AnUnnameableProviderIsNotSerializedEvenWhenExplicit() + { + // a custom provider is still perfectly usable in code; it just cannot be expressed as a string, the + // same way an inbuilt tunnel can be and a custom one cannot + var options = ConfigurationOptions.Parse("localhost"); + options.Defaults = new TestOptionsProvider(".unnameable"); + + Assert.Null(options.Defaults.Name); + Assert.DoesNotContain("defaults=", options.ToString()); + } + + [Theory] + [InlineData("redis-12345.c1.eu-west-1-2.ec2.cloud.redislabs.com")] + [InlineData("contoso.CLOUD.REDISLABS.COM")] // case-insensitive, as the sibling providers are + [InlineData("redis-12345.c1.us-east-1-2.ec2.cloud.redis.io")] // newer routing scheme + [InlineData("contoso.redislabs.com")] // older subscriptions, addressed directly + public void IsMatchOnRedisCloudDomain(string hostName) + { + var epc = new EndPointCollection(new List() { new DnsEndPoint(hostName, 0) }); + var provider = DefaultOptionsProvider.GetProvider(epc); + Assert.IsType(provider); + } + + [Fact] + public void RedisCloudDoesNotInheritTheAzureManagedAssumptions() + { + // the two deployments look similar and are not: AMR is TLS-only with a 7.4 floor, Redis Cloud is + // neither, so copying those two would break plaintext databases and over-claim the server version + var epc = new EndPointCollection(new List() { new DnsEndPoint("contoso.cloud.redislabs.com", 0) }); + var cloud = DefaultOptionsProvider.GetProvider(epc); + var amr = new AzureManagedRedisOptionsProvider(); + + Assert.True(amr.GetDefaultSsl(epc)); + Assert.False(cloud.GetDefaultSsl(epc)); + Assert.Equal(RedisFeatures.v7_4_0, amr.DefaultVersion); + Assert.Equal(DefaultOptionsProvider.BaseDefaultVersion, cloud.DefaultVersion); + + // ...but it does share the parts that are about being a proxied, hosted deployment + Assert.Equal(RedisProtocol.Resp3, cloud.Protocol); + Assert.Equal("", cloud.ConfigurationChannel); + Assert.False(cloud.AbortOnConnectFail); + } + + [Theory] + [InlineData("contoso.redis.azure.net", MaintenanceNotificationMode.Auto)] // AMR asks, pre-emptively + [InlineData("contoso.cloud.redislabs.com", MaintenanceNotificationMode.Auto)] // and Redis Cloud, which emits them + [InlineData("contoso.redis.cache.windows.net", MaintenanceNotificationMode.Disabled)] // classic Azure does not + [InlineData("contoso.example.com", MaintenanceNotificationMode.Disabled)] // and neither does anything else + public void MaintenanceNotificationDefaultPerProvider(string hostName, MaintenanceNotificationMode expected) + { + // Auto rather than Enabled is what makes the pre-emptive default safe: AMR does not emit these yet, + // so until the server side ships the opt-in is refused and the feature simply stays off + var epc = new EndPointCollection(new List() { new DnsEndPoint(hostName, 0) }); + var provider = DefaultOptionsProvider.GetProvider(epc); + Output.WriteLine($"{hostName} -> {provider.GetType().Name}"); + Assert.Equal(expected, provider.MaintenanceNotifications); + + // ...and it arrives through the options, not just off the provider + var options = new ConfigurationOptions { EndPoints = { new DnsEndPoint(hostName, 0) } }; + Assert.Equal(expected, options.MaintenanceNotifications); + } + [Theory] [InlineData(RedisProtocol.Resp2)] [InlineData(RedisProtocol.Resp3)] diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index c8a30030e..0c2d4892f 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using StackExchange.Redis.Availability; using Xunit; @@ -6,6 +6,34 @@ namespace StackExchange.Redis.Tests; public class HealthCheckPolicyUnitTests { + /// + /// Probe traffic must not be counted as work a caller is waiting on, because idleness is what decides + /// whether an endpoint that has left the deployment can be given up - a probe that looks like caller work + /// makes a server appear busy precisely because we are watching it. + /// + /// + /// The other half of this test is what it asserts is *false*: the probe must not become an internal call, + /// which would also change how it is queued - internal calls bypass the backlog. A health check that + /// bypasses the backlog cannot observe a bridge whose queue is not draining, which is exactly the signal + /// geo-redundant failover depends on. + /// + [Fact] + public void ProbeTrafficIsNotCallerFacingButIsRoutedNormally() + { + var flags = new HealthCheckContext(null!, TimeSpan.FromSeconds(1)).ProbeFlags; + Assert.NotEqual(CommandFlags.None, flags); + + var message = Message.Create(-1, flags, RedisCommand.PING); + Assert.True(message.IsProbe); + Assert.False(message.IsCallerFacing); + Assert.False(message.IsInternalCall); + + // and an ordinary command is unaffected in both directions + var ordinary = Message.Create(-1, CommandFlags.None, RedisCommand.PING); + Assert.False(ordinary.IsProbe); + Assert.True(ordinary.IsCallerFacing); + } + [Theory] [InlineData(0, 0, 5, HealthCheckResult.Inconclusive)] // No results yet [InlineData(1, 0, 0, HealthCheckResult.Healthy)] // One success, no more probes diff --git a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs index 09947fa25..7d793f18a 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs @@ -26,6 +26,32 @@ public static class TestConfig public static int MinTimeoutMilliseconds { get; } = int.TryParse(Environment.GetEnvironmentVariable("REDIS_TESTS_MIN_TIMEOUT_MS"), out var ms) && ms > 0 ? ms : 0; + /// + /// A suite-wide maintenance-notification mode, from REDIS_TESTS_MAINT_NOTIFICATIONS + /// (disabled/enabled/auto); null (the default) leaves the library's own + /// default alone. + /// + /// + /// The point is to be able to run the *whole* suite with the opt-in on, rather than to test the feature: + /// most servers we test against never send a notification, and that is the expected outcome - the opt-in + /// must be invisible to everything else. Applied only where a test has not asked for a specific mode. + /// + public static MaintenanceNotificationMode? MaintenanceNotifications { get; } = + Enum.TryParse(Environment.GetEnvironmentVariable("REDIS_TESTS_MAINT_NOTIFICATIONS"), ignoreCase: true, out var mode) + ? mode : null; + + /// + /// Applies , if set. Call before any test-specific override. + /// + public static ConfigurationOptions ApplyMaintenanceDefault(ConfigurationOptions config) + { + if (MaintenanceNotifications is { } mode) + { + config.MaintenanceNotifications = mode; + } + return config; + } + #if NET private static int _db = 17; #else diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 4d27a206b..2276500bd 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -53,6 +53,14 @@ public InProcessTestServer(ITestOutputHelper? log = null, EndPoint? endpoint = n Tunnel = new InProcTunnel(this); } + /// + /// The highest protocol this server will agree to; lowering it lets a test have a HELLO 3 accepted + /// and answered as RESP2, which is the shape a client has to survive without being told about it. + /// + public RedisProtocol MaxProtocolVersion { get; set; } = RedisProtocol.Resp3; + + protected override RedisProtocol MaxProtocol => MaxProtocolVersion; + public Task ConnectAsync(bool withPubSub = true, bool defaultOnly = false, WriteMode writeMode = WriteMode.Default, TextWriter? log = null) => ConnectionMultiplexer.ConnectAsync(GetClientConfig(withPubSub, defaultOnly, writeMode), log); @@ -116,6 +124,7 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default Protocol = TestContext.Current.GetProtocol(), // WriteMode = (BufferedStreamWriter.WriteMode)writeMode, }; + TestConfig.ApplyMaintenanceDefault(config); if (!string.IsNullOrEmpty(Password)) config.Password = Password; config.Ssl = UseSsl; // explicitly, ignore provider defaults if (UseSsl) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs new file mode 100644 index 000000000..33ee7aeb6 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs @@ -0,0 +1,113 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Deciding what a MOVING means: where to go, or that there is nowhere to go. +/// +/// +/// Tested as a pure decision, with the resolver supplied, because the whole thing turns on DNS *changing* - +/// which no in-process fake can arrange. +/// +public class MaintenanceHandoffTests(ITestOutputHelper log) +{ + private static readonly IPAddress Retiring = IPAddress.Parse("10.129.228.140"); + private static readonly IPAddress Replacement = IPAddress.Parse("10.252.90.18"); + private static readonly DnsEndPoint Hostname = new("db.example.cloud.redislabs.com", 13486); + + private static Func> Resolves(params IPAddress[] addresses) + => (_, _) => Task.FromResult(addresses); + + [Fact] + public async Task HostnameThatHasMovedIsRecycled() + { + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.Recycle, decision.Action); + Assert.Equal(new IPEndPoint(Replacement, 13486), decision.Target); + } + + [Fact] + public async Task HostnameThatNeverMovesDoesNothing() + { + // Not a failure: the server closes the socket regardless, the reconnect re-resolves, and the relaxed + // window covers the gap. Measured on a real cluster - DNS updated three seconds *after* the close - so + // this is a normal outcome rather than a defensive branch. + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromMilliseconds(100), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: Resolves(Retiring), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + Assert.Null(decision.Target); + } + + [Fact] + public async Task AddressEndpointWithNoSuccessorHasNothingToDo() + { + // An address cannot be re-resolved, and nothing was named: there is no handoff to make. This is the + // case a cluster deployment would hit, and it is why MOVING must not simply reuse endpoint retirement - + // there would be nothing to retire *to*. + var decision = await MaintenanceHandoff.DecideAsync( + new IPEndPoint(Retiring, 13486), successor: null, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + } + + [Fact] + public async Task NamedSuccessorAsksForAReconfigure() + { + // Never observed on any real deployment - eleven routes, all explicit nulls - so this exists because the + // contract has it, not because it fires. + var successor = new IPEndPoint(Replacement, 13486); + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Retiring), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.Reconfigure, decision.Action); + Assert.Equal(successor, decision.Target); + } + + [Fact] + public async Task UnknownCurrentAddressDoesNothingRatherThanGuessing() + { + // Recycling here would mean accepting whatever DNS says *now*, which for the first several seconds is + // the address being retired - so we would hand off to the node we were told to leave. + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: null, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + } + + [Theory] + [InlineData(15, 0, 1000)] // a generous window: capped at a second, not a tenth of fifteen + [InlineData(2, 0, 200)] // a short one: a tenth, so a 2s window never spends more than 200ms + [InlineData(0, 0, 0)] // "act now" + public void JitterScalesWithTheWindowAndIsCapped(int windowSeconds, int minMilliseconds, int maxMilliseconds) + { + var random = new Random(12345); + for (int i = 0; i < 200; i++) + { + var jitter = MaintenanceHandoff.GetJitter(TimeSpan.FromSeconds(windowSeconds), random); + Assert.InRange(jitter.TotalMilliseconds, minMilliseconds, maxMilliseconds); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs new file mode 100644 index 000000000..0fad1947f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -0,0 +1,833 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Receiving the notifications: can the client see them at all, and does it make sense of the payload. Nothing +/// here asserts a *reaction* - there isn't one yet, deliberately - only that a push frame the server sends +/// arrives as an event with the right contents, and that a malformed one is dropped without collateral damage. +/// +public class MaintenanceNotificationTests(ITestOutputHelper log) +{ + private const int DefaultTimeoutMilliseconds = 5000; + + /// + /// The notifications are RESP3 push frames, so every test here forces RESP3 rather than running per + /// protocol: under RESP2 there is nothing to receive, which is covered by the opt-in tests instead. + /// + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync(ITestOutputHelper log) + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; // must be live, or the test is vacuous + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn, new EventCollector(conn)); + } + + private sealed class EventCollector + { + private readonly ConcurrentQueue _events = new(); + + public EventCollector(IConnectionMultiplexer conn) + => conn.ServerMaintenanceEvent += (_, e) => _events.Enqueue(e); + + public int Count => _events.Count; + + public IReadOnlyList All => _events.ToArray(); + + public async Task NextAsync(int timeoutMilliseconds = DefaultTimeoutMilliseconds) + { + for (int i = 0; i < timeoutMilliseconds / 25; i++) + { + if (_events.TryDequeue(out var next)) return Assert.IsType(next); + await Task.Delay(25); + } + + throw new TimeoutException("No maintenance event was received"); + } + + /// + /// Deliberately proves a negative, so it has to wait out a window in which one could have arrived. + /// + public async Task AssertNoneAsync(int milliseconds = 250) + { + await Task.Delay(milliseconds); + Assert.Empty(All); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating, MaintenanceNotificationType.Migrating)] + [InlineData(MaintenanceNotificationKind.Migrated, MaintenanceNotificationType.Migrated)] + [InlineData(MaintenanceNotificationKind.FailingOver, MaintenanceNotificationType.FailingOver)] + [InlineData(MaintenanceNotificationKind.FailedOver, MaintenanceNotificationType.FailedOver)] + public async Task ShardNotificationIsReceived(MaintenanceNotificationKind sent, MaintenanceNotificationType expected) + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var seq = server.NextMaintenanceSequenceId; + Assert.Equal(1, server.SendShardNotification(null, sent, timeSeconds: 12, shardIds: "[\"shard:1\"]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(expected, evt.NotificationType); + Assert.Equal(seq, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(12), evt.Time); + Assert.Equal("[\"shard:1\"]", evt.Payload); // opaque, carried through verbatim + Assert.Equal(server.DefaultEndPoint, evt.EndPoint); + Assert.Null(evt.NewEndPoint); + } + } + + [Fact] + public async Task MovingCarriesItsSuccessor() + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var target = new IPEndPoint(IPAddress.Loopback, 7999); + Assert.Equal(1, server.SendMoving(null, timeSeconds: 15, newEndpoint: target)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Moving, evt.NotificationType); + Assert.Equal(TimeSpan.FromSeconds(15), evt.Time); + Assert.Equal(target, evt.NewEndPoint); + + // and the deadline is projected forward, which is what a consumer actually wants + Assert.NotNull(evt.StartTimeUtc); + Assert.Equal(evt.ReceivedTimeUtc.AddSeconds(15), evt.StartTimeUtc); + } + } + + [Fact] + public async Task MovingWithNoAddressIsStillReported() + { + // the documented no-address form: a client must handle it whether or not it asked for "none", and the + // intended handling is to reconnect to what it already has - so the event must arrive, not be dropped + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendMoving(null, timeSeconds: 15, newEndpoint: null)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Moving, evt.NotificationType); + Assert.Null(evt.NewEndPoint); + Assert.Null(evt.Payload); + } + } + + [Theory] + [InlineData("?:7002")] // unknown node + [InlineData(":7002")] // node does not know its own address + [InlineData("host-1.example.com:0")] // no port to dial + public async Task MovingWithAPlaceholderYieldsNoEndpoint(string placeholder) + { + // none of these can be dialled, and none of them may be read as "the server that told us" - so the + // event arrives with the raw text and no endpoint, rather than with a plausible-looking wrong one + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MOVING", "1", "15", placeholder)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Null(evt.NewEndPoint); + Assert.Equal(placeholder, evt.Payload); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.SlotMigrating, MaintenanceNotificationType.SlotMigrating)] + [InlineData(MaintenanceNotificationKind.SlotMigrated, MaintenanceNotificationType.SlotMigrated)] + public async Task SlotNotificationIsReceived(MaintenanceNotificationKind sent, MaintenanceNotificationType expected) + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotNotification(null, sent, slots: "123,456,789-1000")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(expected, evt.NotificationType); + Assert.Equal("123,456,789-1000", evt.Payload); + Assert.Null(evt.Time); // these carry no duration + } + } + + [Fact] + public async Task AllDigitSlotListIsNotMistakenForADuration() + { + // why the type decides whether a time is expected, rather than the content: a single-slot list is + // indistinguishable from a duration by inspection + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, slots: "123")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal("123", evt.Payload); + Assert.Null(evt.Time); + } + } + + [Fact] + public async Task IntegersMayArriveAsStrings() + { + // "accept $ or : for integers" - the contract says so explicitly, and SendRawPush writes bulk strings + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MIGRATING", "42", "7", "[\"shard:2\"]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(42, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(7), evt.Time); + } + } + + [Fact] + public async Task NegativeTimeIsPreserved() + { + // a connection that joins mid-window is told what is left of it, which can be negative: that means + // "act now", so it must not be clamped or rejected + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: -3)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(TimeSpan.FromSeconds(-3), evt.Time); + Assert.Null(evt.StartTimeUtc); // nothing sensible to project + } + } + + [Fact] + public async Task LowercaseTypeNamesAreRecognized() + { + // no specification is careful about casing, so neither are we + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "failing_over", "5", "9", "[]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.FailingOver, evt.NotificationType); + } + } + + [Fact] + public async Task TrailingElementsAreTolerated() + { + // forwards compatibility is a stated requirement: a future field must not cost us the notification + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MIGRATING", "8", "20", "[\"shard:3\"]", "something-new", "and-another")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Migrating, evt.NotificationType); + Assert.Equal(8, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(20), evt.Time); + Assert.Equal("[\"shard:3\"]", evt.Payload); + } + } + + [Fact] + public async Task NestedSlotMigrationsAreRead() + { + // the shape the shipped clients read: [type, seq, [[source, target, slots], ...]]. We were dropping + // this until the prior-art cross-check, because the parser rejected non-scalar elements + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7000", "127.0.0.1:7001", "0-99"), + ("127.0.0.1:7002", "127.0.0.1:7003", "1000,2000-2500"), + ])); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrated, evt.NotificationType); + Assert.Equal(2, evt.SlotMigrations.Count); + + var first = evt.SlotMigrations[0]; + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 7000), first.Source); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 7001), first.Target); + Assert.Equal(new SlotRange(0, 99), Assert.Single(first.Slots)); + + var second = evt.SlotMigrations[1]; + Assert.Equal(2, second.Slots.Count); + Assert.Equal(new SlotRange(1000, 1000), second.Slots[0]); + Assert.Equal(new SlotRange(2000, 2500), second.Slots[1]); + Assert.Equal("1000,2000-2500", second.RawSlots); + } + } + + [Fact] + public async Task CapturedEnterpriseFramesAreUnderstood() + { + // Captured from a real Redis Cloud QA endpoint (Enterprise 8.6.2, OSS cluster API) during an actual + // slot migration on 2026-08-27. Byte for byte, except that the node addresses are rewritten into the + // private range - the lengths are preserved, so the $20 and $18 counts below are still the real ones: + // + // >3\r\n$10\r\nSMIGRATING\r\n:18\r\n$9\r\n8892-8991\r\n + // >3\r\n$9\r\nSMIGRATED\r\n:19\r\n*1\r\n*3\r\n$20\r\n10.129.228.140:13486\r\n + // $18\r\n10.252.90.18:13486\r\n$9\r\n8892-8991\r\n + // + // Things this pins, each of which was an assumption before: the type name arrives as a *bulk* string; + // the sequence number as a RESP integer rather than a string; neither cluster notification carries a + // time element; SMIGRATING's slots are a flat string; and SMIGRATED nests an array *of* triplets - one + // here - rather than a single flat triple. The fake emits this shape, so this test is the real frame. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, "8892-8991", sequenceId: 18); + + var migrating = await events.NextAsync(); + log.WriteLine(migrating.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrating, migrating.NotificationType); + Assert.Equal(18, migrating.SequenceId); + Assert.Null(migrating.Time); // no time element on the wire + Assert.Equal("8892-8991", migrating.Payload); + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [("10.129.228.140:13486", "10.252.90.18:13486", "8892-8991")], sequenceId: 19); + + var migrated = await events.NextAsync(); + log.WriteLine(migrated.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrated, migrated.NotificationType); + Assert.Equal(19, migrated.SequenceId); + Assert.Null(migrated.Time); + + var migration = Assert.Single(migrated.SlotMigrations); + Assert.Equal(new IPEndPoint(IPAddress.Parse("10.129.228.140"), 13486), migration.Source); + Assert.Equal(new IPEndPoint(IPAddress.Parse("10.252.90.18"), 13486), migration.Target); + Assert.Equal(new SlotRange(8892, 8991), Assert.Single(migration.Slots)); + } + } + + [Fact] + public async Task CapturedMovingFrameIsUnderstood() + { + // Captured from the same deployment during a maintenance_mode scenario, byte for byte: + // + // >4\r\n$6\r\nMOVING\r\n:0\r\n:15\r\n_\r\n + // + // Four elements: type, sequence *zero*, a 15-second window, and an explicit RESP3 null for the + // address - so "no replacement given, reconnect the way you connected". The proxy then closed the + // socket, which is the point of MOVING: you are told to move, and then the connection goes away. + // + // The sequence number is zero because this was the first event of that chain, not because MOVING is + // special - which is the point: zero is a legal value, so dedup tracks "have we seen one" as a separate + // bit rather than treating a stored zero as unset. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendMoving(null, timeSeconds: 15, newEndpoint: null, sequenceId: 0); + + var moving = await events.NextAsync(); + log.WriteLine(moving.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + Assert.Equal(0, moving.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(15), moving.Time); + Assert.Null(moving.NewEndPoint); // the null is the documented "use what you already have" + Assert.Null(moving.Payload); + + // and the window it opened is what covers the reconnect that follows the socket closing + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "MOVING should have relaxed timeouts"); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating, MaintenanceNotificationKind.Migrated, "27")] + [InlineData(MaintenanceNotificationKind.FailingOver, MaintenanceNotificationKind.FailedOver, "21")] + public async Task CapturedShardNotificationsAreUnderstood( + MaintenanceNotificationKind opening, + MaintenanceNotificationKind closing, + string shard) + { + // Captured from the same deployment, byte for byte, running the fault injector's maintenance_mode and + // failover scenarios: + // + // >4\r\n$9\r\nMIGRATING\r\n:0\r\n:2\r\n$6\r\n["27"]\r\n + // >3\r\n$8\r\nMIGRATED\r\n:1\r\n$6\r\n["27"]\r\n + // >4\r\n$12\r\nFAILING_OVER\r\n:0\r\n:2\r\n$6\r\n["21"]\r\n + // >3\r\n$11\r\nFAILED_OVER\r\n:1\r\n$6\r\n["21"]\r\n + // + // Both families have the same shape, and it is the shape the parser assumed: the *opening* + // notification carries a time and the *closing* one has no time element at all - which is why + // CarriesTime is keyed on the notification type rather than on sniffing whether the third element + // looks like a number. The shard list is a stringified JSON array of id strings, quotes included, and + // is carried through opaquely: nothing a client does depends on which shard it was. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var shards = $"[\"{shard}\"]"; + server.SendShardNotification(null, opening, timeSeconds: 2, shardIds: shards, sequenceId: 0); + + var open = await events.NextAsync(); + log.WriteLine(open.RawMessage ?? "(none)"); + Assert.Equal(TimeSpan.FromSeconds(2), open.Time); + Assert.Equal(shards, open.Payload); + Assert.Equal(0, open.SequenceId); + + // note 2 seconds is what the server actually said, and is shorter than the relaxed timeout floor; + // the window is clamped up to the floor rather than being honoured literally + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, $"{opening} should have relaxed timeouts"); + + server.SendShardNotification(null, closing, timeSeconds: null, shardIds: shards, sequenceId: 1); + + var closed = await events.NextAsync(); + log.WriteLine(closed.RawMessage ?? "(none)"); + Assert.Null(closed.Time); // three elements on the wire: no time to read + Assert.Equal(shards, closed.Payload); + Assert.Equal(1, closed.SequenceId); + } + } + + [Fact] + public async Task OneLogicalEventFromEveryNodeRaisesOneEvent() + { + // Observed on the real deployment: every node broadcasts a given event, and all the copies carry the + // same sequence number - the id identifies the event, not the delivery. So a two-node cluster delivers + // one migration twice. Both deliveries have to be *acted on*, because the timeout relaxation is + // per-server, but the consumer wants one callback rather than one per proxy. + var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + GetHost(server.DefaultEndPoint, out var port); + var second = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + server.Migrate((RedisKey)"elsewhere", second); // owns something, so the client connects to it + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var events = new EventCollector(conn); + using (server) + { + var muxer = (IInternalConnectionMultiplexer)conn; + Assert.True( + await Poll.UntilAsync(() => + { + var endpoints = conn.GetEndPoints(); + return endpoints.Length == 2 && endpoints.All(ep => muxer.GetServerEndPoint(ep).MaintenanceNotificationsActive); + }), + "both nodes should be connected and opted in, or there is only one delivery and this proves nothing"); + + // one send, two clients: this is the fan-out, from the server side + var delivered = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + Assert.Equal(2, delivered); + + // both servers relax - that is the internal work that has to happen per connection... + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().All(ep => muxer.GetServerEndPoint(ep).IsMaintenanceRelaxed)), + "every node that was told should have relaxed its own timeouts"); + + // ...and the consumer sees it once + Assert.Equal(1, events.Count); + var evt = await events.NextAsync(); + Assert.Equal(5, evt.SequenceId); + log.WriteLine($"{events.Count} event(s) from {delivered} deliveries: {evt.RawMessage}"); + Assert.Contains(evt.EndPoint, conn.GetEndPoints()); // whichever told us first + } + } + + /// + /// The catch-up channel: Redis Enterprise retains the most recent shard-scoped *completion* and replays it + /// to each connection that opts in, so a client that connects after a disruption still learns it happened. + /// + /// + /// Observed on RS 8.0.22 (2026-08-28), arriving coalesced into the same TCP read as the +OK for the + /// opt-in, at ~16 ms. These tests key on the *cause* of that rather than the framing: the frame is flushed + /// the instant the opt-in is processed, and our opt-in is pipelined early, so a retained frame always + /// arrives while the connection is still handshaking. + /// + public class Retention(ITestOutputHelper log) + { + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( + ITestOutputHelper log, + Action beforeConnect) + { + var server = new InProcessTestServer(log); + beforeConnect(server); // the event happens *before* anybody connects: that is the whole point + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn, new EventCollector(conn)); + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrated)] + [InlineData(MaintenanceNotificationKind.FailedOver)] + public async Task RetainedCompletionIsReplayedToANewConnection(MaintenanceNotificationKind kind) + { + var (server, conn, events) = await ConnectAsync(log, s => + s.SendShardNotification(null, kind, timeSeconds: null, shardIds: "[\"27\"]", sequenceId: 5)); + + using (server) + await using (conn) + { + // the collector is attached *after* connecting, so this asserts the frame was handled rather + // than that the event fired - which is why the relaxed window is the observable here + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "the replayed completion should have opened the post-event tail"); + Assert.Equal(MaintenanceNotificationTypeFor(kind), endpoint.ActiveMaintenanceType); + log.WriteLine($"{kind} replayed; relaxed = {endpoint.IsMaintenanceRelaxed}"); + + // ...and it must not have disturbed the handshake it arrived in the middle of, which is the + // other half of what this test is for: a push frame interleaved with our own handshake + // replies must be dispatched out-of-band rather than matched against one of them + Assert.True(conn.IsConnected); + Assert.True(await conn.GetDatabase().PingAsync() >= TimeSpan.Zero); + GC.KeepAlive(events); + } + } + + private static MaintenanceNotificationType MaintenanceNotificationTypeFor(MaintenanceNotificationKind kind) => kind switch + { + MaintenanceNotificationKind.Migrated => MaintenanceNotificationType.Migrated, + MaintenanceNotificationKind.FailedOver => MaintenanceNotificationType.FailedOver, + _ => MaintenanceNotificationType.None, + }; + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating)] + [InlineData(MaintenanceNotificationKind.FailingOver)] + [InlineData(MaintenanceNotificationKind.Moving)] + [InlineData(MaintenanceNotificationKind.SlotMigrating)] + [InlineData(MaintenanceNotificationKind.SlotMigrated)] + public async Task StartersAndSlotScopedEventsAreNotRetained(MaintenanceNotificationKind kind) + { + // The design property, asserted as a negative: the catch-up channel can only ever say "a + // disruption ended", never "one is starting". Nothing that demands action is replayed, so a + // reconnecting client cannot be told to move by a stale frame - which is what makes the MOVING + // handoff safe from replay by construction rather than by a guard. + var (server, conn, events) = await ConnectAsync(log, s => + { + if (kind == MaintenanceNotificationKind.SlotMigrated) + { + s.SendSlotMigrations(null, kind, [("127.0.0.1:7000", "127.0.0.1:7001", "50-60")], sequenceId: 5); + } + else + { + s.SendShardNotification(null, kind, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + } + }); + + using (server) + await using (conn) + { + await events.AssertNoneAsync(); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.False(endpoint.IsMaintenanceRelaxed, $"{kind} must not be retained, so nothing should have relaxed"); + } + } + + [Fact] + public async Task RetentionReplacesRatherThanAccumulates() + { + // "most recent completion", not a queue: a connection sees at most one of these however many + // events went past, which is why the client side needs no window or ring for the catch-up case + var (server, conn, events) = await ConnectAsync(log, s => + { + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"1\"]", sequenceId: 5); + s.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, null, "[\"2\"]", sequenceId: 6); + }); + + using (server) + await using (conn) + { + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed); + Assert.Equal(MaintenanceNotificationType.FailedOver, endpoint.ActiveMaintenanceType); // the later one + GC.KeepAlive(events); + } + } + + [Fact] + public async Task RetentionCanBeTurnedOff() + { + // not every deployment retains, and a test of the no-retention case should not have to reason + // about which notification kinds happen to be retained + var (server, conn, events) = await ConnectAsync(log, s => + { + s.RetainCompletions = false; + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"27\"]", sequenceId: 5); + }); + + using (server) + await using (conn) + { + await events.AssertNoneAsync(); + Assert.False(((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).IsMaintenanceRelaxed); + } + } + } + + [Theory] + [InlineData(null)] // the measured shape: the close arrives *after* the announced window + [InlineData(300)] // a less generous proxy: the close beats the window it announced + public async Task MovingClosesTheConnectionAndWeRecover(int? closeDelayMilliseconds) + { + // The defining half of MOVING, which no test covered until now: we are told to move, and then the + // socket goes away. Measured on RS (2026-08-28): the close landed at +18.4s and +16.6s against a + // declared 15s window, so the window is a floor with slack rather than a deadline - which is why the + // default case here lets the fake overshoot, and why nothing below asserts on the window expiring. + // The short case is the defensive one: a client that treats the announced window as guaranteed fails + // it. What must hold either way is that the connection comes back and the relaxed window opened by the + // notification is what covers the reconnect. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.MovingClosesConnection = true; + server.MovingCloseDelay = closeDelayMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : null; + server.SendMoving(null, timeSeconds: 1, newEndpoint: null, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "the window has to be open before the socket dies"); + + // the client must reconnect on its own; nothing here asks it to. Note the probe has to tolerate + // throwing rather than returning false: a command issued in the window between the socket dying + // and the reconnect completing raises, and that is the state being polled *out* of + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 15_000), + "the multiplexer should have re-established itself after the close"); + + log.WriteLine($"close delay {closeDelayMilliseconds?.ToString() ?? "announced+slack"}: reconnected, relaxed = {endpoint.IsMaintenanceRelaxed}"); + } + } + + [Fact] + public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() + { + // The point of D6: today a MOVING is survivable because the socket eventually closes and we reconnect, + // but the announced window goes unused. Here nothing closes the connection - MovingClosesConnection is + // deliberately left off - so the *only* thing that can replace it is our own handoff. + // + // The named-successor branch is what is exercised, because the in-process transport has no socket and so + // no current address for the DNS branch to compare against; the deciding half of that branch is covered + // by MaintenanceHandoffTests against a supplied resolver. What this proves is the acting half: drain, + // drop, and come back. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var optInsBefore = server.TotalMaintenanceOptIns; + Assert.Equal(1, optInsBefore); // the handshake's own opt-in, so the increment below means something + + // A handoff has to be *visible*: the replacement raises ConnectionRestored, so reporting nothing for + // the drop would leave a consumer tracking connection state with an unpaired restore. + var failures = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (failures) failures.Add(e.FailureType); + }; + + server.SendMoving(null, timeSeconds: 2, newEndpoint: server.DefaultEndPoint, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + Assert.Equal(server.DefaultEndPoint, moving.NewEndPoint); + + // a fresh connection re-sends the opt-in, which is how a recycle is visible from the server's side + Assert.True( + await Poll.UntilAsync(() => server.TotalMaintenanceOptIns > optInsBefore, timeoutMilliseconds: 15_000), + "the handoff should have replaced the connection without the server closing it"); + + log.WriteLine($"opt-ins: {optInsBefore} -> {server.TotalMaintenanceOptIns}"); + + Assert.True( + await Poll.UntilAsync( + () => + { + lock (failures) return failures.Contains(ConnectionFailureType.MaintenanceHandoff); + }, + timeoutMilliseconds: 5_000), + "the recycle should be reported as a MaintenanceHandoff rather than silently"); + + lock (failures) + { + log.WriteLine($"reported: {string.Join(", ", failures)}"); + + // and *only* as that: reporting it as a socket failure would put planned maintenance into + // everybody's fault dashboards + Assert.DoesNotContain(ConnectionFailureType.SocketFailure, failures); + Assert.DoesNotContain(ConnectionFailureType.SocketClosed, failures); + } + + // ...and the replacement is usable, which is the only outcome a caller cares about + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 15_000), + "the client should be serving commands on the replacement connection"); + } + } + + [Fact] + public async Task MalformedTripletIsSkippedNotFatal() + { + // one bad entry must not lose the migrations we could have applied - the same choice go-redis makes + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7000", "127.0.0.1:7001", "not-a-slot-list"), // kept, but with no parsed slots + ("127.0.0.1:7002", "?", "50-60"), // an unnameable target, kept with a null Target + ("127.0.0.1:7004", "127.0.0.1:7005", "70"), + ]); + + var evt = await events.NextAsync(); + Assert.Equal(3, evt.SlotMigrations.Count); + Assert.Empty(evt.SlotMigrations[0].Slots); + Assert.Equal("not-a-slot-list", evt.SlotMigrations[0].RawSlots); // raw form survives + Assert.Null(evt.SlotMigrations[1].Target); + Assert.Equal(new SlotRange(50, 60), Assert.Single(evt.SlotMigrations[1].Slots)); + Assert.Equal(new SlotRange(70, 70), Assert.Single(evt.SlotMigrations[2].Slots)); + } + } + + [Fact] + public async Task MissingSequenceIdStillReportsTheNotification() + { + // stricter-than-shipped-clients was the bug here: go-redis length-checks these at two elements and + // reads no sequence number at all, so dropping a frame for want of one loses a real disruption + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + // two elements, with an unreadable seq - the floor go-redis works to + Assert.True(server.SendRawPush(null, "MIGRATING", "not-a-number") > 0); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Migrating, evt.NotificationType); + } + } + + [Theory] + [InlineData("NOT_A_REAL_KIND", "1", "5")] // a type we don't know + [InlineData("NOT_A_REAL_KIND")] // ...and one with nothing else at all + public async Task MalformedNotificationIsDroppedAndTheConnectionSurvives(params string[] parts) + { + // the important half of this feature's safety: an unparseable push frame must be consumed and + // forgotten. If it fell through to the command matcher it would steal the next command's reply, and + // everything after that would be answering the wrong question + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.True(server.SendRawPush(null, parts) > 0); + await events.AssertNoneAsync(); + + // the connection is not merely alive but *in sync*: each reply is the answer to its own command + var db = conn.GetDatabase(); + for (int i = 0; i < 10; i++) + { + await db.StringSetAsync($"maint-sync-{i}", i); + } + + for (int i = 0; i < 10; i++) + { + Assert.Equal(i, (int)await db.StringGetAsync($"maint-sync-{i}")); + } + } + } + + [Fact] + public async Task NotificationsArriveInterleavedWithCommands() + { + // a push can land at any point, including between a command and its reply; nothing may be lost or + // mismatched either way + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var db = conn.GetDatabase(); + var pending = new List>(); + for (int i = 0; i < 50; i++) + { + await db.StringSetAsync($"maint-interleave-{i}", i); + pending.Add(db.StringGetAsync($"maint-interleave-{i}")); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: i); + } + + var values = await Task.WhenAll(pending); + Assert.Equal(Enumerable.Range(0, 50), values.Select(x => (int)x)); + + // and every notification arrived + for (int i = 0; i < 50; i++) + { + await events.NextAsync(); + } + + log.WriteLine($"{values.Length} commands and 50 notifications, all accounted for"); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs new file mode 100644 index 000000000..7da72d32d --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -0,0 +1,251 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// The client half of the opt-in: what we send during handshake, and what we make of the answer. The server +/// half is . +/// +[RunPerProtocol] +public class MaintenanceOptInClientTests(ITestOutputHelper log) +{ + private static InProcessTestServer CreateServer(ITestOutputHelper log) => new(log); + + private static ConfigurationOptions Config(InProcessTestServer server, MaintenanceNotificationMode mode) + { + var config = server.GetClientConfig(defaultOnly: true); + config.MaintenanceNotifications = mode; + return config; + } + + private static bool IsActive(IConnectionMultiplexer conn, InProcessTestServer server) + => ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).MaintenanceNotificationsActive; + + private static List OptedIn(InProcessTestServer server) + { + var found = new List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } + + /// + /// Asserts that Enabled refused: either the connect threw, or it returned a connection that does + /// not stay usable. + /// + /// + /// Both outcomes are the same refusal, and which one a caller sees is a race: the reconcile records the + /// failure from the handshake-completion path, which can land either side of ConnectAsync deciding + /// it has a connection. Asserting only the throw made this flaky on a two-core runner, and asserting the + /// throw is not the point - not being left with a working connection is. + /// + private static async Task AssertRefusedAsync(ConfigurationOptions config, ITestOutputHelper log) + { + ConnectionMultiplexer? conn = null; + try + { + conn = await ConnectionMultiplexer.ConnectAsync(config); + } + catch (RedisConnectionException ex) + { + log.WriteLine($"refused at connect: {ex.Message}"); + return; + } + + await using (conn) + { + var endpoint = conn.GetEndPoints().Single(); + var unusable = await Poll.UntilAsync(() => !conn.GetServer(endpoint).IsConnected); + log.WriteLine($"connected, then unusable: {unusable}"); + Assert.True(unusable, "the connection should not have stayed usable"); + } + } + + [Fact] + public async Task HandshakeOptsInWhenAuto() + { + using var server = CreateServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Auto)); + + var resp3 = TestContext.Current.IsResp3(); + log.WriteLine($"protocol: {TestContext.Current.GetProtocol()}, opted in: {OptedIn(server).Count}"); + + // RESP3-only by design: under RESP2 the server could accept the request and then never deliver + // anything, so we don't ask - and under Auto that is simply the feature being off + Assert.Equal(resp3, IsActive(conn, server)); + Assert.Equal(resp3 ? 1 : 0, OptedIn(server).Count); + + if (resp3) + { + // a bare ON, so the server chooses; nothing here invents an endpoint type + Assert.Null(Assert.Single(OptedIn(server)).MovingEndpointType); + } + + // ...and the connection is entirely usable either way + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task DisabledSendsNothing() + { + using var server = CreateServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Disabled)); + + Assert.Empty(OptedIn(server)); + Assert.False(IsActive(conn, server)); + } + + [Fact] + public async Task DefaultIsOff() + { + // the library default has to be Disabled: OSS Redis, Valkey and Garnet don't know the subcommand, and + // an unsolicited error reply on every connection would be a poor first impression + using var server = CreateServer(log); + var config = server.GetClientConfig(defaultOnly: true); // untouched, so whatever the provider says + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Equal(MaintenanceNotificationMode.Disabled, config.MaintenanceNotifications); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.UnknownSubcommand)] + [InlineData(MaintenanceNotificationSupport.Disabled)] + public async Task AutoToleratesAServerThatRefuses(MaintenanceNotificationSupport support) + { + // the whole point of Auto: ask, and carry on regardless. This is what lets the entire test suite run + // with the opt-in on against servers that will never send a notification + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Auto)); + + Assert.False(IsActive(conn, server)); + Assert.Empty(OptedIn(server)); + Assert.True(conn.GetServer(server.DefaultEndPoint).IsConnected); + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task EnabledFailsAgainstAServerThatRefuses() + { + // Enabled is a requirement, not a preference: a connection that silently won't deliver notifications + // is not the connection that was asked for + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.MaintenanceNotifications = MaintenanceNotificationSupport.UnknownSubcommand; + + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task AutoIsOffWhenTheServerDowngradesToResp2() + { + // the case that makes this RESP3-only: the server accepts the opt-in and then answers HELLO as RESP2, + // so nothing would ever arrive. We asked, the server said OK, and we still treat it as off + using var server = CreateServer(log); + server.MaxProtocolVersion = RedisProtocol.Resp2; + var config = Config(server, MaintenanceNotificationMode.Auto); + config.Protocol = RedisProtocol.Resp3; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Single(OptedIn(server)); // the server took it... + Assert.False(IsActive(conn, server)); // ...and we know better than to believe it + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task EnabledFailsWhenTheServerDowngradesToResp2() + { + using var server = CreateServer(log); + server.MaxProtocolVersion = RedisProtocol.Resp2; + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.Protocol = RedisProtocol.Resp3; + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task EnabledFailsWhenResp2WasOurOwnChoice() + { + // no exemption for a contradiction we configured ourselves: requiring a RESP3-only feature over a + // RESP2 connection cannot be honoured, and half-honouring it silently is what Enabled exists to avoid + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.Protocol = RedisProtocol.Resp2; + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task AutoIsHappyOnResp2() + { + // the counterpart: Auto is best-effort, so an explicit RESP2 just means the feature is off + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.Protocol = RedisProtocol.Resp2; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Empty(OptedIn(server)); + Assert.False(IsActive(conn, server)); + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task OptInIsReArmedOnReconnect() + { + // per-connection state, so a reconnect that didn't re-send it would leave us silently unsubscribed + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.AllowSimulateConnectionFailure = true; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(IsActive(conn, server)); + + var before = server.TotalMaintenanceOptIns; + conn.GetServer(server.DefaultEndPoint).SimulateConnectionFailure(SimulatedFailureType.All); + await UntilCondition(() => server.TotalMaintenanceOptIns > before); + + log.WriteLine($"opt-ins: {before} -> {server.TotalMaintenanceOptIns}"); + Assert.True(server.TotalMaintenanceOptIns > before, "the opt-in should be sent again on the new connection"); + + // ...and then wait for *our* side of it. The count above is the server having processed the opt-in, + // whereas IsActive is us having processed its reply - a beat later, so asserting it directly is a race + // that only shows up when the runner is starved of cores. + await UntilCondition(() => IsActive(conn, server)); + Assert.True(IsActive(conn, server), "the feature should be live again on the replacement connection"); + } + + private static async Task UntilCondition(System.Func condition, int timeoutMilliseconds = 5000) + { + for (int i = 0; i < timeoutMilliseconds / 50 && !condition(); i++) + { + await Task.Delay(50); + } + } + + private static async Task Set(IConnectionMultiplexer conn) + { + var db = conn.GetDatabase(); + await db.StringSetAsync("maint-optin", "value"); + return await db.StringGetAsync("maint-optin"); + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs new file mode 100644 index 000000000..c44296df0 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs @@ -0,0 +1,322 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Timeout relaxation: a server that announces a disruption gets its command timeouts raised for the duration, +/// as a floor and never a reduction. The window arithmetic is asserted through internals rather than by waiting +/// out real seconds - the interesting properties are "does it open, extend, close, and expire correctly", and a +/// test that sleeps for 30s to watch a cap expire is a test nobody runs. +/// +public class MaintenanceRelaxationTests(ITestOutputHelper log) +{ + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + Action? configure = null) + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + configure?.Invoke(config); + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + private static ServerEndPoint Endpoint(IConnectionMultiplexer conn, InProcessTestServer server) + => ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + + /// + /// The notification arrives on the connection's read loop, so a test that sends one has to let it land. + /// + private static async Task UntilRelaxedAsync(ServerEndPoint endpoint, bool expected, int timeoutMilliseconds = 5000) + { + for (int i = 0; i < timeoutMilliseconds / 25 && endpoint.IsMaintenanceRelaxed != expected; i++) + { + await Task.Delay(25); + } + return endpoint.IsMaintenanceRelaxed; + } + + [Fact] + public async Task NoWindowMeansNoChange() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + Assert.False(endpoint.IsMaintenanceRelaxed); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + } + } + + [Fact] + public async Task OpeningNotificationRelaxesTimeouts() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + + Assert.True(await UntilRelaxedAsync(endpoint, true), "should be relaxed"); + + // 10s relaxed against a 5s configured timeout + Assert.Equal(10_000, endpoint.GetEffectiveTimeoutMilliseconds(5_000)); + } + } + + [Fact] + public async Task RelaxationIsAFloorAndNeverAReduction() + { + // a caller with a generous timeout keeps it; this is explicit in the contract + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.Equal(60_000, endpoint.GetEffectiveTimeoutMilliseconds(60_000)); + } + } + + [Fact] + public async Task ClosingNotificationLeavesThePostEventTail() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + await Task.Delay(250); // let it land; the tail means "still relaxed", so there is no flag to await + + // the server said it finished, and we are *still* relaxed - because that is when every other + // client that got the same notification comes back + Assert.True(endpoint.IsMaintenanceRelaxed, "the post-event tail should still be running"); + Assert.Equal(10_000, endpoint.GetEffectiveTimeoutMilliseconds(5_000)); + } + } + + [Fact] + public async Task ClosingNotificationEndsItWhenThereIsNoTail() + { + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false), "should have ended"); + } + } + + [Fact] + public async Task WindowExpiresOnItsOwn() + { + // the cap and the ordinary deadline share a mechanism; a one-second floor lets us watch it expire + // without a test that sleeps for half a minute + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero; + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + + // asks for 20s, capped to 1s + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.False(await UntilRelaxedAsync(endpoint, false), "the cap should have ended it"); + } + } + + [Fact] + public async Task NoTailAfterACapExpiry() + { + // if the window ended on the cap we never learned the event finished, so extending past the backstop + // would defeat the backstop + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.FromSeconds(30); // would be very visible + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.False(await UntilRelaxedAsync(endpoint, false), "no tail should follow a capped window"); + } + } + + [Fact] + public async Task ShorterNotificationDoesNotCutAnOpenWindowShort() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + + // a long window, then a short one on top: the short one must not shorten it + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 25); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 1); + await Task.Delay(1500); + + Assert.True(endpoint.IsMaintenanceRelaxed, "the longer window should still be in force"); + } + } + + [Fact] + public async Task ReplayedSequenceIdIsIgnored() + { + // our own invention, so kept conservative: an id we have already acted on cannot extend a window + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero; + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + var seq = server.NextMaintenanceSequenceId; + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + Assert.False(await UntilRelaxedAsync(endpoint, false)); + + // replaying the same id must not reopen anything + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20, sequenceId: seq); + await Task.Delay(250); + Assert.False(endpoint.IsMaintenanceRelaxed, "a replayed id should not reopen the window"); + + // ...but a genuinely new one still does + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true), "a new id should still work"); + } + } + + [Fact] + public async Task RelaxationDoesNotSuppressConnectionFailure() + { + // the regression that would matter most: relaxation must touch command timeouts only. If it reached + // keep-alive or failure detection, a server that died mid-maintenance would linger for the window - + // turning a latency mitigation into an availability regression + var (server, conn) = await ConnectAsync(log, config => + { + config.AllowSimulateConnectionFailure = true; + config.MaintenanceRelaxedTimeout = TimeSpan.FromMinutes(5); // absurdly generous, deliberately + config.MaintenanceRelaxedWindowMax = TimeSpan.FromMinutes(5); + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 300); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + var failures = 0; + conn.ConnectionFailed += (_, _) => Interlocked.Increment(ref failures); + conn.GetServer(server.DefaultEndPoint).SimulateConnectionFailure(SimulatedFailureType.All); + + for (int i = 0; i < 100 && Volatile.Read(ref failures) == 0; i++) + { + await Task.Delay(25); + } + + log.WriteLine($"connection failures observed: {Volatile.Read(ref failures)}"); + Assert.True(Volatile.Read(ref failures) > 0, "a dead connection must still be noticed during a relaxed window"); + } + } + + [Theory] + [InlineData(true, MaintenanceNotificationType.Migrating)] + [InlineData(false, MaintenanceNotificationType.None)] + public async Task TimeoutReportsWhetherMaintenanceWasInForce(bool announce, MaintenanceNotificationType expected) + { + // "timeout" and "timeout during an announced migration" call for very different reactions from + // whoever reads the log, so the fault says which it was + var (server, conn) = await ConnectAsync(log, config => + { + config.AsyncTimeout = 200; + config.SyncTimeout = 200; + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(30); + config.HeartbeatInterval = TimeSpan.FromMilliseconds(100); + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + if (announce) + { + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + } + + server.SetLatency(TimeSpan.FromSeconds(5)); // longer than any timeout in play + var ex = await Assert.ThrowsAsync(() => conn.GetDatabase().StringGetAsync("maint-fault")); + log.WriteLine($"{ex.MaintenanceType}: {ex.Message}"); + Assert.Equal(expected, ex.MaintenanceType); + + // and it reaches the circuit-breaker/retry surface the same way + var context = new Availability.FaultContext(ex); + Assert.Equal(expected, context.MaintenanceType); + + server.SetLatency(TimeSpan.Zero); + } + } + + [Fact] + public async Task CommandsStillWorkThroughAWindow() + { + // relaxation is invisible to anything that is not timing out + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + var db = conn.GetDatabase(); + for (int i = 0; i < 20; i++) + { + await db.StringSetAsync($"relax-{i}", i); + } + for (int i = 0; i < 20; i++) + { + Assert.Equal(i, (int)await db.StringGetAsync($"relax-{i}")); + } + + // including the synchronous path, which has its own re-wait loop + Assert.Equal(7, (int)db.StringGet("relax-7")); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs new file mode 100644 index 000000000..5cddbb1eb --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs @@ -0,0 +1,400 @@ +using System; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Server; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Non-parallel deliberately. Several of these wait out a jittered refresh, and the retirement one depends on +/// the doomed server being *idle* - and IsIdle counts outstanding work, which a heartbeat ping in flight +/// supplies. On a loaded machine those complete slowly enough that pruning is starved, which is a real property +/// of the policy rather than a flaw in the test: see the design notes on why a usage-based grace rule was +/// dropped for exactly this reason. +/// +/// Reacting to a completed slot migration by re-reading the topology, rather than waiting to be told by a +/// -MOVED. Asserted from the server's side - did another CLUSTER command actually arrive - because +/// that is the thing the fleet pays for, and the thing a scoping bug would multiply. +/// +/// +[Collection(NonParallelCollection.Name)] +public class MaintenanceTopologyRefreshTests(ITestOutputHelper log) +{ + /// + /// Counts inbound CLUSTER commands, so a test can tell a refresh happened from the outside. + /// + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands, _subscribeCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public int SubscribeCommands => Volatile.Read(ref _subscribeCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0) + { + var command = request.GetString(0); + if (string.Equals(command, "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + else if (string.Equals(command, "ssubscribe", StringComparison.OrdinalIgnoreCase) + || string.Equals(command, "subscribe", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _subscribeCommands); + } + } + + return base.Execute(client, in request); + } + } + + private static async Task<(CountingServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + Action? configure = null, + MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + configure?.Invoke(server); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = mode; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + /// + /// The jitter is up to a second, so a refresh is not immediate by design. + /// + private static Task UntilRefreshedAsync(CountingServer server, int before) + => Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 5000); + + [Fact] + public async Task SlotsMovingAwayFromUsTriggersARefresh() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", "0-99"), + ]); + + Assert.True(await UntilRefreshedAsync(server, before), "a refresh should follow slots leaving us"); + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + } + } + + [Fact] + public async Task SlotsMovingBetweenOtherNodesDoesNotTriggerARefresh() + { + // the whole herd argument: every node reports the same movements, so most notifications are about + // somebody else. Refreshing on those means every client in the fleet re-reads topology whenever any + // shard moves anywhere + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7100", "127.0.0.1:7101", "0-99"), + ("127.0.0.1:7102", "127.0.0.1:7103", "100-199"), + ]); + + await Task.Delay(2000); // longer than the jitter, so "not yet" cannot be mistaken for "never" + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task SourceIsMatchedByAnnouncedIdentityNotJustAddress() + { + // A node answers to its address *and* its announced hostname, and the delta may name either, so the + // scoping test resolves through the identity map rather than comparing endpoints as text. Announced + // before connecting, because the handshake's own topology read is what registers the identity - a + // hostname set afterwards is not known to us until something re-reads it. + const string Hostname = "node-1.redis.example.com"; + var (server, conn) = await ConnectAsync(log, s => s.SetHostname(s.DefaultEndPoint, Hostname)); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"{Hostname}:{port}", $"127.0.0.1:{port + 1}", "0-99"), + ]); + + Assert.True(await UntilRefreshedAsync(server, before), "the hostname form should resolve to us"); + } + } + + [Fact] + public async Task ProxyStyleCompletionDoesNotTriggerARefresh() + { + // MIGRATED/FAILED_OVER arrive in proxied deployments addressed as a single endpoint, where there is no + // topology for a refresh to learn - so they get relaxation and nothing else + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + + await Task.Delay(2000); + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task ShardedSubscriptionOnAMovedSlotIsReEstablished() + { + // mostly belt-and-braces: the server also sends an unsolicited SUNSUBSCRIBE for this, which we already + // act on. What this adds is being pre-emptive when SMIGRATED lands first, and covering the case where + // the unsubscribe never arrives - where the only other symptom is messages silently stopping + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("resub-channel"); + await sub.SubscribeAsync(channel, (_, _) => { }); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + log.WriteLine($"channel hashes to slot {slot}"); + + var before = server.SubscribeCommands; + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{slot}"), + ]); + + Assert.True( + await Poll.UntilAsync(() => server.SubscribeCommands > before), + "the sharded subscription should have been re-established"); + } + } + + [Fact] + public async Task ShardedSubscriptionOnAnUnaffectedSlotIsLeftAlone() + { + // knowing *which* slots moved is the advantage over the unsolicited-unsubscribe path: only the + // affected channels are touched, rather than everything subscribed on this server + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("untouched-channel"); + await sub.SubscribeAsync(channel, (_, _) => { }); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + var otherSlot = slot == 0 ? 1 : 0; + var before = server.SubscribeCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{otherSlot}"), + ]); + + await Task.Delay(2000); + log.WriteLine($"subscribe commands: {before} -> {server.SubscribeCommands} (slot {slot} vs moved {otherSlot})"); + Assert.Equal(before, server.SubscribeCommands); + } + } + + [Fact] + public async Task OrdinaryPubSubIsNotSlotBoundAndIsLeftAlone() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + await sub.SubscribeAsync(RedisChannel.Literal("plain-channel"), (_, _) => { }); + + var before = server.SubscribeCommands; + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", "0-16383"), // everything moves + ]); + + await Task.Delay(2000); + log.WriteLine($"subscribe commands: {before} -> {server.SubscribeCommands}"); + Assert.Equal(before, server.SubscribeCommands); + } + } + + [Theory] + [InlineData(MaintenanceNotificationMode.Disabled)] // only the unsolicited SUNSUBSCRIBE path can act + [InlineData(MaintenanceNotificationMode.Enabled)] // both paths see the same migration + public async Task RealMigrationRecoversTheSubscriptionWithoutStorming(MaintenanceNotificationMode mode) + { + // Until the fake announced its own migrations this could not be tested at all, and it is the case that + // matters: a real slot migration produces *both* an unsolicited SUNSUBSCRIBE (ordinary cluster + // behaviour, sent to every subscriber) and SMIGRATED (only to clients that opted in). Both paths lead + // to ResubscribeToServer, so the question is whether the subscription ends up established exactly + // once rather than twice or not at all. + var (server, conn) = await ConnectAsync(log, mode: mode); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var doomed = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("migrating-channel"); + var received = 0; + await sub.SubscribeAsync(channel, (_, _) => Interlocked.Increment(ref received)); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + var before = server.SubscribeCommands; + + server.NotifyOnMigrate = true; + server.Migrate(slot, doomed); // the whole realistic sequence, from one call + + Assert.True(await Poll.UntilAsync(() => server.SubscribeCommands > before), "should have resubscribed"); + await Task.Delay(2000); // and settle, so a second resubscribe would show up + + var resubscribes = server.SubscribeCommands - before; + log.WriteLine($"{mode}: {resubscribes} (re)subscribe command(s) after a real migration"); + + // The property is boundedness, not an absolute count: the unsolicited-unsubscribe path has its own + // pre-existing retry-and-redirect behaviour. Measured at 4 with notifications off and 5 with them + // on - the extra one being the fallback acting on a subscription that path left attached to + // nothing, which is the feature working. What must not happen is one attempt per notification. + // Bounded, not exact: the unsolicited-unsubscribe path retries and follows redirects, and how + // many of those land depends on timing (measured 4 unloaded, 7 on a constrained runner). The + // property is that the count does not scale with notifications - one migration, a handful of + // attempts - so the bound is deliberately loose rather than pinned to a number that will drift. + Assert.InRange(resubscribes, 1, 15); + + // Delivery is asserted by publishing *repeatedly*, because pub/sub is fire and forget: a message + // published while the subscription is still in flux is simply dropped. Losing messages during the + // tremor is expected; never delivering again is not, and one publish cannot tell those apart. + var delivered = await Poll.UntilAsync( + () => + { + if (Volatile.Read(ref received) > 0) return true; + conn.GetSubscriber().Publish(channel, "hello"); + return Volatile.Read(ref received) > 0; + }, + timeoutMilliseconds: 10_000, + pollMilliseconds: 250); + + log.WriteLine($"{mode}: delivered after migration = {delivered}"); + Assert.True(delivered, "the subscription should deliver again once things settle"); + } + } + + [Fact] + public async Task NodeThatLeavesTheClusterIsRetired() + { + // This is also the regression test for idleness counting *caller* work only. Before that, the + // reconfigure's own probes to the departed node piled into its backlog (~170 per pass, unbounded), so + // it looked busy because we were looking for it, and could never be retired. + + // The narrowed form of D5's "retire endpoints serving no slots". Serving nothing is *not* the + // condition: a node still listed in CLUSTER NODES is a live member that may be given slots again, and + // dropping its connection would be churn (go-redis does not). Having *left* the cluster is the + // condition, and the notification-driven refresh is what makes us notice. + EndPoint doomed = null!; + var (server, conn) = await ConnectAsync(log, configure: s => + { + GetHost(s.DefaultEndPoint, out var p); + doomed = s.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, p + 1)); + s.Migrate((RedisKey)"leaving", doomed); // owns something, so it is discovered at connect + }); + + using (server) + await using (conn) + { + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + GetHost(server.DefaultEndPoint, out var basePort); + var third = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, basePort + 2)); + + // hand its slot back and then remove it from the cluster outright + server.NotifyOnMigrate = true; + server.Migrate((RedisKey)"leaving", server.DefaultEndPoint); + Assert.True(server.RemoveNode(doomed), "the node should have been removed from the fake"); + + // Pruning wants several consecutive generations of absence. Driving those by *awaiting* topology + // passes rather than by sending notifications and sleeping: the notification path is covered by + // the tests above, and depending on it here would make this test wait out a jittered refresh per + // generation - which is what made it fail under a two-core runner. What is being tested is the + // retirement, so drive the generations deterministically. + // Retirement also requires the server to be *idle*, so a busy moment can defer it past a given + // pass - which is why this drives passes until it happens rather than asserting after a fixed + // count. Bounded, so a regression fails rather than hangs. + GC.KeepAlive(third); + var mux = (ConnectionMultiplexer)conn; + for (int i = 0; i < 20 && conn.GetEndPoints().Contains(doomed); i++) + { + await mux.ReconfigureAsync(first: false, reconfigureAll: true, log: null, blame: null, cause: $"test-generation-{i}"); + await Task.Delay(50); // retirement drains before removing, so give it a moment to complete + + } + + log.WriteLine($"endpoints: {string.Join(", ", conn.GetEndPoints().Select(x => x.ToString()))}"); + Assert.DoesNotContain(doomed, conn.GetEndPoints()); + } + } + + [Fact] + public async Task RefreshIsCoalescedRatherThanRepeated() + { + // A burst of notifications must not be a burst of topology reads. Note ReconfigureIfNeeded's own + // coalescing is *not* what achieves this: it declines only while a refresh is actually in flight, and + // the jitter spreads a burst out far enough that each pass finishes before the next starts. This test + // is what caught that, and why the coalescing happens before the delay rather than after. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + for (int i = 0; i < 10; i++) + { + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{i * 10}-{(i * 10) + 9}"), + ]); + } + + Assert.True(await UntilRefreshedAsync(server, before)); + await Task.Delay(2000); // let any stragglers land + + // a refresh reads both SLOTS and NODES, so allow a small multiple - the point is that ten + // notifications did not produce ten topology passes + var added = server.ClusterCommands - before; + log.WriteLine($"{added} cluster command(s) for 10 notifications"); + Assert.InRange(added, 1, 8); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs index 748589e2b..83ded0d7b 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Threading.Tasks; @@ -107,6 +107,6 @@ public void LiteralsResolveToExpectedStorageKinds() // only two storage flavors should ever appear; a third means something new crept in (e.g. a numeric // literal => Int64), which we want to notice Assert.Equal(2, byKind.Count); - Assert.Equal(22, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes + Assert.Equal(23, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes } } diff --git a/tests/StackExchange.Redis.Tests/TestBase.cs b/tests/StackExchange.Redis.Tests/TestBase.cs index c59039478..505f2bb4e 100644 --- a/tests/StackExchange.Redis.Tests/TestBase.cs +++ b/tests/StackExchange.Redis.Tests/TestBase.cs @@ -454,7 +454,7 @@ public static ConnectionMultiplexer CreateDefault( log ??= localLog = new StringWriter(); try { - var config = ConfigurationOptions.Parse(configuration); + var config = TestConfig.ApplyMaintenanceDefault(ConfigurationOptions.Parse(configuration)); if (disabledCommands != null && disabledCommands.Length != 0) { config.CommandMap = CommandMap.Create([.. disabledCommands], false); diff --git a/toys/MaintenanceWatch/MaintenanceWatch.csproj b/toys/MaintenanceWatch/MaintenanceWatch.csproj new file mode 100644 index 000000000..b400a6ced --- /dev/null +++ b/toys/MaintenanceWatch/MaintenanceWatch.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + + $(NoWarn);SER010 + + + + + + + diff --git a/toys/MaintenanceWatch/Program.cs b/toys/MaintenanceWatch/Program.cs new file mode 100644 index 000000000..f75aacdec --- /dev/null +++ b/toys/MaintenanceWatch/Program.cs @@ -0,0 +1,115 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; +using StackExchange.Redis.Maintenance; + +// Watches a real deployment for server-native maintenance notifications and prints what we made of them. +// +// The point of this tool is *evidence*: that we asked for notifications, that they arrive, and that each one +// was understood - so every event prints both our parsed view and the raw payload it came from, and any +// disagreement between the two is visible rather than inferred. +// +// dotnet run --project toys/MaintenanceWatch -- "my-endpoint:6379,user=...,password=..." +// +// Notes: +// - RESP3 is required; this forces it rather than relying on the endpoint's provider defaults. +// - The opt-in mode is forced to Auto, so an endpoint that does not support notifications still connects. +// - Nothing here reacts to a notification; observing is the whole job. +if (args.Length == 0) +{ + Console.Error.WriteLine("usage: MaintenanceWatch [--enabled]"); + return 1; +} + +var config = ConfigurationOptions.Parse(args[0]); +config.Protocol = RedisProtocol.Resp3; +config.MaintenanceNotifications = args.Contains("--enabled") + ? MaintenanceNotificationMode.Enabled // refuse to run if the server will not deliver + : MaintenanceNotificationMode.Auto; +config.AbortOnConnectFail = false; + +// note ToString() includes the password by default; the library's own logging masks it (see +// LoggerExtensions), but this line is ours, so it has to ask +Console.WriteLine($"connecting: {config.ToString(includePassword: false)}"); +Console.WriteLine($"defaults provider: {config.Defaults}"); +Console.WriteLine($"maintNotifications={config.MaintenanceNotifications}, relaxed={config.MaintenanceRelaxedTimeout.TotalSeconds}s," + + $" windowMax={config.MaintenanceRelaxedWindowMax.TotalSeconds}s, postEvent={config.MaintenancePostEventRelaxedDuration.TotalSeconds}s"); + +// the connect log carries the negotiation half: whether the opt-in was sent, and how the server answered +await using var conn = await ConnectionMultiplexer.ConnectAsync(config, Console.Out); + +var count = 0; +conn.ServerMaintenanceEvent += (_, e) => +{ + var n = Interlocked.Increment(ref count); + Console.WriteLine(); + Console.WriteLine($"=== maintenance event #{n} at {DateTime.UtcNow:HH:mm:ss.fff}Z ==="); + Console.WriteLine($" raw: {e.RawMessage}"); + + if (e is not PushMaintenanceEvent push) + { + // e.g. the Azure pub/sub channel; not what this tool is for, but worth seeing rather than hiding + Console.WriteLine($" (not a push notification: {e.GetType().Name})"); + return; + } + + Console.WriteLine($" type: {push.NotificationType} seq: {push.SequenceId} from: {push.EndPoint}"); + Console.WriteLine($" time: {(push.Time is { } t ? $"{t.TotalSeconds}s" : "(none)")}" + + $" startsAt: {(push.StartTimeUtc is { } at ? $"{at:HH:mm:ss}Z" : "(n/a)")}"); + + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + // null is a documented outcome, not a parse failure: reconnect to what you already have + Console.WriteLine($" moving to: {push.NewEndPoint?.ToString() ?? "(no address given)"}"); + } + + if (push.Payload is { } payload) + { + Console.WriteLine($" payload: {payload}"); + } + + foreach (var migration in push.SlotMigrations) + { + var slots = migration.Slots.Count == 0 + ? $"(unparsed: '{migration.RawSlots}')" + : string.Join(",", migration.Slots.Select(x => x.From == x.To ? $"{x.From}" : $"{x.From}-{x.To}")); + Console.WriteLine($" slots {slots}: {migration.Source?.ToString() ?? "?"} -> {migration.Target?.ToString() ?? "?"}"); + } +}; + +// a fault during an announced disruption says so, which is the other half of "we understood it" +conn.ErrorMessage += (_, e) => Console.WriteLine($"[error] {e.EndPoint}: {e.Message}"); +conn.ConnectionFailed += (_, e) => Console.WriteLine($"[failed] {e.EndPoint}: {e.FailureType} {e.Exception?.Message}"); +conn.ConnectionRestored += (_, e) => Console.WriteLine($"[restored] {e.EndPoint}"); + +Console.WriteLine(); +Console.WriteLine("watching; a little traffic keeps the connection interesting. Ctrl+C to stop."); + +var db = conn.GetDatabase(); +var key = $"maintenance-watch:{Guid.NewGuid():N}"; +while (true) +{ + try + { + await db.StringSetAsync(key, (RedisValue)DateTime.UtcNow.Ticks); + _ = await db.StringGetAsync(key); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // note RedisTimeoutException derives from TimeoutException, *not* RedisException - so a + // `catch (RedisException)` would silently miss exactly the case this tool exists to show + // MaintenanceType is the payoff here: "timeout" versus "timeout during an announced failover" + var maintenance = ex switch + { + RedisTimeoutException timeout => timeout.MaintenanceType, + RedisConnectionException connection => connection.MaintenanceType, + _ => MaintenanceNotificationType.None, + }; + Console.WriteLine($"[command] {ex.GetType().Name}: {ex.Message}" + + (maintenance == MaintenanceNotificationType.None ? "" : $" <- during {maintenance}")); + } + + await Task.Delay(1000); +} diff --git a/toys/StackExchange.Redis.Server/RedisClient.Output.cs b/toys/StackExchange.Redis.Server/RedisClient.Output.cs index 0e9396a5a..b0a2b21d6 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.Output.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.Output.cs @@ -27,6 +27,30 @@ private readonly struct VersionedResponse(TypedRedisValue value, RedisProtocol p private readonly Channel _replies = Channel.CreateUnbounded(s_replyChannelOptions); + private TypedRedisValue _deferred = TypedRedisValue.Nil; + + /// + /// Queues a frame to be sent *after* the reply to the command currently being executed. + /// + /// + /// Needed because the read loop enqueues a command's reply once Execute has returned, so a handler + /// that calls directly puts its frame *before* its own reply. A real server does + /// the opposite - the retained maintenance notification arrives immediately after the +OK for the + /// opt-in, in the same TCP segment - and the ordering is the whole point of modelling this. + /// + public void AddOutboundAfterReply(in TypedRedisValue message) + { + FlushDeferredOutbound(); // one slot is enough for any real case; do not lose an earlier frame + _deferred = message; + } + + internal void FlushDeferredOutbound() + { + var pending = _deferred; + _deferred = TypedRedisValue.Nil; + if (!pending.IsNil) AddOutbound(pending); + } + public void AddOutbound(in TypedRedisValue message) { if (message.IsNil) @@ -139,6 +163,9 @@ public async Task WriteOutputAsync(PipeWriter writer) } catch (Exception ex) { + // this used to vanish into the pipe: a serialization bug in a fake server's *outbound* path + // presents as "the client never received it", with a reconnect covering the tracks. Log it + Node?.Server?.Log($"[{this}] write loop faulted: {ex.Message}"); await writer.CompleteAsync(ex); } } diff --git a/toys/StackExchange.Redis.Server/RedisClient.cs b/toys/StackExchange.Redis.Server/RedisClient.cs index 69fa5437a..bed7b00f0 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.cs @@ -157,7 +157,12 @@ public void Dispose() } private int _activeSlot = ServerSelectionStrategy.NoSlot; - internal void ResetAfterRequest() => _activeSlot = ServerSelectionStrategy.NoSlot; + internal void ResetAfterRequest() + { + _activeSlot = ServerSelectionStrategy.NoSlot; + FlushDeferredOutbound(); // after the reply for this command, which the read loop has now queued + } + public virtual void OnKey(in RedisKey key, KeyFlags flags) { if ((flags & KeyFlags.NoSlotCheck) == 0 & node.CheckCrossSlot) diff --git a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs index 14eda7143..51530f5a9 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.Net; +using System.Threading; +using System.Threading.Tasks; using RESPite; using RESPite.Messages; @@ -54,6 +57,96 @@ public enum MaintenanceNotificationKind }; private int _maintenanceSequence; + private int _maintenanceOptIns; + private (MaintenanceNotificationKind Kind, long Sequence, string ShardIds)? _retainedCompletion; + + /// + /// Whether the server keeps the most recent shard-scoped *completion* and replays it to each + /// connection that opts in, as Redis Enterprise does. + /// + /// + /// Observed on RS 8.0.22 (2026-08-28), and the boundary is sharp: MIGRATED and + /// FAILED_OVER are retained; MIGRATING, FAILING_OVER, MOVING, + /// SMIGRATING and SMIGRATED are not. The characterisation that fits all seven is the + /// completion of a *shard-scoped* event - the two that carry an affected-shards list - so + /// SMIGRATED (a slot-range triple) and MOVING (an endpoint) are excluded even though + /// SMIGRATED is also a completion. + /// + /// The consequence is a design property worth stating: the catch-up channel can only ever say "a + /// disruption ended", never "one is starting". Nothing that demands action is replayed, so a + /// reconnecting client cannot be told to move by a stale frame. + /// + /// + /// Retained "most recent, replaced not accumulated" - one frame, never a queue - so a connection sees + /// at most one of these however many events went past. + /// + /// + public bool RetainCompletions { get; set; } = true; + + /// + /// Whether this notification is one of the two the server retains for replay. + /// + private static bool IsRetainedCompletion(MaintenanceNotificationKind kind) + => kind is MaintenanceNotificationKind.Migrated or MaintenanceNotificationKind.FailedOver; + + /// + /// Replays the retained completion, if any, to a client that has just opted in. + /// + /// + /// Deliberately *after* the +OK (see ), and with + /// the original sequence id rather than a fresh one: the id identifies the event, and a client using it + /// for dedup has to be able to recognize a completion it has already seen. + /// + private void ReplayRetainedCompletion(RedisClient client) + { + if (!RetainCompletions || _retainedCompletion is not { } retained) return; + + client.AddOutboundAfterReply(BuildShardNotification(retained.Kind, null, retained.Sequence, null, retained.ShardIds)); + Log($"[{client}] replayed retained {GetName(retained.Kind)} (seq {retained.Sequence})"); + } + + /// + /// How many times a client has opted in, across the lifetime of the server. Per-client state goes away + /// with the client, so this is what a reconnect test can measure. + /// + public int TotalMaintenanceOptIns => Volatile.Read(ref _maintenanceOptIns); + + internal void OnMaintenanceOptIn() => Interlocked.Increment(ref _maintenanceOptIns); + + /// + /// Whether announces itself the way a real server does, rather + /// than only moving the slot in this model. + /// + /// + /// Off by default, because plenty of tests use Migrate purely to arrange a topology and would + /// not expect a notification to arrive mid-arrangement. Turn it on to exercise the *sequence* a client + /// really sees - which is the only way to test how the notification-driven path and the unsolicited + /// SUNSUBSCRIBE path interact, since both fire for the same migration. + /// + public bool NotifyOnMigrate { get; set; } + + /// + /// Announces a slot migration the way a real server does: the shard notifications either side of it, + /// and an unsolicited sunsubscribe to any client subscribed to a sharded channel that has just + /// moved away. + /// + /// + /// Note the two signals are independent. The notifications only reach clients that opted in, whereas + /// the unsubscribe is ordinary cluster behaviour and reaches every subscriber - so a client can + /// legitimately see one, the other, or both, and the order is a server implementation detail. That is + /// exactly the interaction worth being able to reproduce here. + /// + private void AnnounceMigration(int hashSlot, Node from, Node to) + { + var slots = hashSlot.ToString(System.Globalization.CultureInfo.InvariantCulture); + SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, slots); + + var dropped = ForAllClients(hashSlot, static (client, slot) => client.UnsubscribeMigratedSlot(slot)); + if (dropped != 0) Log($"unsubscribed {dropped} sharded subscription(s) for migrated slot {hashSlot}"); + + SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [($"{from.Host}:{from.Port}", $"{to.Host}:{to.Port}", slots)]); + } /// /// The sequence id given to the next notification, unless one is supplied explicitly. The contract does @@ -69,14 +162,97 @@ public enum MaintenanceNotificationKind /// /// The number of clients the notification was sent to. public int SendMoving(RedisClient client, int timeSeconds, EndPoint newEndpoint, int? sequenceId = null) - => Send(client, MaintenanceNotificationKind.Moving, timeSeconds, sequenceId, newEndpoint, null); + { + var recipients = MovingClosesConnection ? new List() : null; + Action onSent = recipients is null ? null : recipients.Add; + var count = Send(client, MaintenanceNotificationKind.Moving, timeSeconds, sequenceId, newEndpoint, null, onSent); + + if (recipients is { Count: > 0 }) + { + // The half of MOVING that defines it: the socket goes away - and it takes every other + // connection to the same node with it (see MovingClosesConnection). + var delay = MovingCloseDelay ?? TimeSpan.FromSeconds(Math.Max(timeSeconds, 0)) + MeasuredCloseSlack; + _ = CloseAfterAsync(CollectSiblings(recipients), delay); + } + + return count; + } + + /// + /// Whether then closes the affected connections, as a real proxy does. + /// + /// + /// Note the blast radius is the *node*, not the connection. Measured on RS (2026-08-28) with four + /// connections to one node differing only in handshake - two opted in, one RESP3 without the opt-in, + /// one RESP2 - all four closed simultaneously, and only the two that had opted in were warned. So this + /// is endpoint retirement rather than socket recycling, and a connection that did not opt in gets no + /// warning at all before it dies: an argument for opting in on every connection rather than one. + /// + public bool MovingClosesConnection { get; set; } + + /// + /// How much later than the announced window the close actually arrives, by default. + /// + /// + /// Measured at +3.4s and +1.6s against a declared 15s grace, i.e. the announced window behaves as a + /// floor with slack rather than a deadline. Tests should assert that a client acts *within* the window, + /// never that the socket survives to the end of it. + /// + public TimeSpan MeasuredCloseSlack { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// How long after MOVING the connections are closed; defaults to the announced window plus + /// , which is what a real proxy was measured doing. + /// + /// + /// Set it shorter than the announced window to exercise a proxy less generous than the one measured - + /// a client that treats the window as guaranteed rather than as a budget fails that case. It cannot + /// usefully be set to zero: the close would race the delivery of the notification itself, which is not + /// something any real timing produces. + /// + public TimeSpan? MovingCloseDelay { get; set; } + + /// + /// Expands the notified connections to every connection sharing a node with them. + /// + private List CollectSiblings(List notified) + { + var nodes = new HashSet(); + foreach (var client in notified) nodes.Add(client.Node); + + var all = new List(); + ForAllClients( + all, + (client, list) => + { + if (nodes.Contains(client.Node)) list.Add(client); + return 0; + }); + return all; + } + + private async Task CloseAfterAsync(List clients, TimeSpan delay) + { + if (delay > TimeSpan.Zero) await Task.Delay(delay).ConfigureAwait(false); + foreach (var client in clients) + { + Log($"[{client}] closing connection after MOVING"); + client.Kill(); + } + } /// /// Sends one of the shard-scoped notifications. is a remaining-time - /// delta and may legitimately be zero or negative for a connection that arrived mid-window. + /// delta and may legitimately be zero or negative for a connection that arrived mid-window; pass + /// null to omit the element entirely, which is what a real server does for the *closing* + /// notifications - captured from Enterprise 8.6.2: + /// + /// >4 $12 FAILING_OVER :0 :2 $6 ["21"] + /// >3 $11 FAILED_OVER :1 $6 ["21"] + /// /// /// The number of clients the notification was sent to. - public int SendShardNotification(RedisClient client, MaintenanceNotificationKind kind, int timeSeconds, string shardIds = null, int? sequenceId = null) + public int SendShardNotification(RedisClient client, MaintenanceNotificationKind kind, int? timeSeconds, string shardIds = null, int? sequenceId = null) => Send(client, kind, timeSeconds, sequenceId, null, shardIds); /// @@ -87,6 +263,48 @@ public int SendShardNotification(RedisClient client, MaintenanceNotificationKind public int SendSlotNotification(RedisClient client, MaintenanceNotificationKind kind, string slots, int? sequenceId = null) => Send(client, kind, null, sequenceId, null, slots); + /// + /// Sends SMIGRATED in its nested form - [type, seq, [[source, target, slots], ...]] - + /// which is what the shipped clients read (see the topic README's prior art; go-redis reads exactly + /// this shape and redis-py models the same nesting). + /// + /// + /// Note the sender is not implicitly the source of anything: every node reports the same movements, so + /// a test can and should exercise a delta that does not involve this server at all. + /// + /// The number of clients the notification was sent to. + public int SendSlotMigrations(RedisClient client, MaintenanceNotificationKind kind, (string Source, string Target, string Slots)[] migrations, int? sequenceId = null) + { + // one sequence id for the notification, however many clients it is delivered to - it identifies the + // event, not the delivery, which is what a real server does and what client-side dedup relies on + var seq = sequenceId ?? Interlocked.Increment(ref _maintenanceSequence); + return Dispatch(client, Build, requireOptIn: true); + + TypedRedisValue Build() + { + // Rent at every level: Recycle() recurses, so a Standalone child inside a pooled parent gets + // handed to the pool it did not come from ("The buffer is not associated with this pool") + var frame = TypedRedisValue.Rent(3, out var span, RespPrefix.Push); + // bulk, not simple: that is what a real server sends. Captured from Enterprise 8.6.2: + // >3 $9 SMIGRATED :19 *1[ *3[ $20 $18 $9 ] ] + span[0] = TypedRedisValue.BulkString(GetName(kind)); + span[1] = TypedRedisValue.Integer(seq); + + var outer = TypedRedisValue.Rent(migrations.Length, out var outerSpan, RespPrefix.Array); + for (int i = 0; i < migrations.Length; i++) + { + var triplet = TypedRedisValue.Rent(3, out var inner, RespPrefix.Array); + inner[0] = TypedRedisValue.BulkString(migrations[i].Source); + inner[1] = TypedRedisValue.BulkString(migrations[i].Target); + inner[2] = TypedRedisValue.BulkString(migrations[i].Slots); + outerSpan[i] = triplet; + } + + span[2] = outer; + return frame; + } + } + /// /// Sends an arbitrary push frame to a client, for the cases a well-formed notification cannot express: /// an unknown type, a malformed payload, extra trailing elements. @@ -94,12 +312,17 @@ public int SendSlotNotification(RedisClient client, MaintenanceNotificationKind /// The number of clients the frame was sent to. public int SendRawPush(RedisClient client, params string[] parts) { - var frame = TypedRedisValue.Rent(parts.Length, out var span, RespPrefix.Push); - for (int i = 0; i < parts.Length; i++) + return Dispatch(client, Build, requireOptIn: false); + + TypedRedisValue Build() { - span[i] = TypedRedisValue.BulkString(parts[i]); + var frame = TypedRedisValue.Rent(parts.Length, out var span, RespPrefix.Push); + for (int i = 0; i < parts.Length; i++) + { + span[i] = TypedRedisValue.BulkString(parts[i]); + } + return frame; } - return Dispatch(client, frame, requireOptIn: false); } private int Send( @@ -108,16 +331,35 @@ private int Send( int? timeSeconds, int? sequenceId, EndPoint newEndpoint, - string extra) + string extra, + Action onSent = null) { // [type, seqID, ...] - the sequence id is an integer, which is precisely why these frames could // not be treated as pub/sub: element 1 is not a channel name + var seq = sequenceId ?? System.Threading.Interlocked.Increment(ref _maintenanceSequence); + + // the retention is server-wide and replaces rather than accumulates, as a real one does; note it + // records what was *sent*, so a test arranges it by sending the completion, not by poking state + if (IsRetainedCompletion(kind)) _retainedCompletion = (kind, seq, extra); + + return Dispatch(client, Build, requireOptIn: true, onSent); + + TypedRedisValue Build() => BuildShardNotification(kind, timeSeconds, seq, newEndpoint, extra); + } + + private static TypedRedisValue BuildShardNotification( + MaintenanceNotificationKind kind, + int? timeSeconds, + long seq, + EndPoint newEndpoint, + string extra) + { int count = 2 + (timeSeconds.HasValue ? 1 : 0) + (newEndpoint is not null || kind == MaintenanceNotificationKind.Moving ? 1 : 0) + (extra is not null ? 1 : 0); var frame = TypedRedisValue.Rent(count, out var span, RespPrefix.Push); int index = 0; - span[index++] = TypedRedisValue.SimpleString(GetName(kind)); - span[index++] = TypedRedisValue.Integer(sequenceId ?? System.Threading.Interlocked.Increment(ref _maintenanceSequence)); + span[index++] = TypedRedisValue.BulkString(GetName(kind)); // bulk, as a real server sends + span[index++] = TypedRedisValue.Integer(seq); if (timeSeconds.HasValue) span[index++] = TypedRedisValue.Integer(timeSeconds.GetValueOrDefault()); if (kind == MaintenanceNotificationKind.Moving) { @@ -132,27 +374,38 @@ private int Send( } if (extra is not null) span[index] = TypedRedisValue.BulkString(extra); - return Dispatch(client, frame, requireOptIn: true); + return frame; } - private int Dispatch(RedisClient client, in TypedRedisValue frame, bool requireOptIn) + /// + /// Sends a notification to one client or to every opted-in client, building the frame per recipient. + /// + /// + /// The factory is called once per recipient rather than the frame being built once and shared, because + /// each client's write loop recycles what it wrote: a shared frame is returned to the pool by the first + /// writer and the second one faults with "Array element cannot be nil", killing that connection. Every + /// broadcast therefore used to reach exactly one client, which is invisible in a single-node test and + /// silently made multi-node fan-out untestable. + /// + private int Dispatch(RedisClient client, Func frameFactory, bool requireOptIn, Action onSent = null) { if (client is not null) { - client.AddOutbound(frame); + client.AddOutbound(frameFactory()); + onSent?.Invoke(client); return 1; } // a real server sends only to connections that asked, so sending to all means all *opted-in*. // Counting the sends rather than the clients visited: the Action overload of ForAllClients returns // one per client regardless, which would report every connection as a recipient - var copy = frame; return ForAllClients( requireOptIn, (target, gated) => { if (gated && !target.MaintenanceNotifications) return 0; - target.AddOutbound(copy); + target.AddOutbound(frameFactory()); + onSent?.Invoke(target); return 1; }); } diff --git a/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs b/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs index 7778ed63b..1498ce1c0 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs @@ -62,7 +62,9 @@ protected virtual TypedRedisValue SPublish(RedisClient client, in RedisRequest r var slot = ServerSelectionStrategy.GetClusterSlot((byte[])channel); if (!node.HasSlot(slot)) KeyMovedException.Throw(slot); - PublishPair pair = new(channel, request.GetValue(2)); + // note the node: without it pair.Node is null, ReferenceEquals never matches, and sharded publish + // silently delivers to nobody - which is how it behaved until 2026-08-26 + PublishPair pair = new(channel, request.GetValue(2), node); int count = ForAllClients(pair, static (client, pair) => ReferenceEquals(client.Node, pair.Node) ? client.Publish(pair.Channel, pair.Value) : 0); return TypedRedisValue.Integer(count); @@ -280,6 +282,38 @@ private Regex BuildGlob(RedisChannel channel) return new Regex(re, RegexOptions.CultureInvariant); } + /// + /// Drops this client's sharded subscriptions whose channel hashes into , + /// pushing the unsolicited sunsubscribe a real server sends when a slot moves away. + /// + /// How many subscriptions were dropped. + internal int UnsubscribeMigratedSlot(int hashSlot) + { + var subs = SubscriptionsIfAny; + if (subs is null) return 0; + + List affected = null; + lock (subs) + { + foreach (var pair in subs) + { + var channel = pair.Key; + if (channel.IsSharded && ServerSelectionStrategy.GetClusterSlot((byte[])channel) == hashSlot) + { + (affected ??= new()).Add(channel); + } + } + } + + if (affected is null) return 0; + foreach (var channel in affected) + { + Unsubscribe(channel); // removes it *and* sends the push, exactly as a client-issued one would + } + + return affected.Count; + } + internal void Unsubscribe(RedisChannel channel) { var subs = SubscriptionsIfAny; diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index af2bd890d..88d20a262 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -112,6 +112,12 @@ public bool Migrate(int hashSlot, EndPoint to) throw new KeyNotFoundException($"Unable to remove slot {hashSlot} from old owner"); } target.AddSlot(hashSlot); + + if (NotifyOnMigrate) + { + AnnounceMigration(hashSlot, pair.Value, target); + } + return true; } } @@ -311,6 +317,38 @@ public EndPoint AddEmptyNode(EndPoint endpoint, NodeFlags flags = NodeFlags.None return endpoint; } + /// + /// Removes a node from the cluster entirely, as CLUSTER FORGET does: it stops appearing in + /// CLUSTER SLOTS and in CLUSTER NODES, and any name it answered to stops resolving. + /// + /// + /// Refuses while it still owns slots, which is also what a real cluster does - a node has to have its + /// slots migrated away before it can be forgotten, and allowing it here would produce a topology with + /// unowned slots that says nothing useful about client behaviour. + /// + /// This exists to distinguish the two conditions a client has to tell apart: a node that currently + /// serves nothing is still a cluster member and may be given slots again, whereas a node that has + /// *left* is one whose connection should be given up. + /// + /// + public bool RemoveNode(EndPoint endpoint) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (!_nodes.TryGetValue(endpoint, out var node)) return false; + if (node.HasAnySlot) throw new InvalidOperationException($"Node still owns slots: {Format.ToString(endpoint)}"); + + if (!_nodes.TryRemove(endpoint, out _)) return false; + + // and every alias that pointed at it, or a stale name would resolve to a node that is gone + foreach (var pair in _aliases) + { + if (ReferenceEquals(pair.Value, node)) _aliases.TryRemove(pair.Key, out _); + } + + Log($"node removed from the cluster: {Format.ToString(endpoint)}"); + return true; + } + public EndPoint AddEmptyNode(NodeFlags flags = NodeFlags.None) { EndPoint endpoint; @@ -722,6 +760,9 @@ public enum MaintenanceNotificationSupport // explicitly valid and means "use the server defaults", so the parameter list is optional and // unrecognized parameters are an error rather than something to ignore - the client is asking the // server to do something specific, and silently not doing it would be worse than refusing + private static bool IsKeyword(in RedisRequest request, int index, string keyword) + => string.Equals(request.GetString(index), keyword, StringComparison.OrdinalIgnoreCase); + [RedisCommand(-3, nameof(RedisCommand.CLIENT), "maint_notifications", LockFree = true)] protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, in RedisRequest request) { @@ -733,9 +774,12 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i return TypedRedisValue.Error("ERR maintenance notifications are disabled on this server"); } + // keywords are matched case-insensitively, as a real server does; clients differ here (go-redis + // sends lowercase, we send the same uppercase form we use for every other keyword), and a fake + // that only accepted one of them would fail a client for something a real server allows bool on; - if (request.IsString(2, "on"u8)) on = true; - else if (request.IsString(2, "off"u8)) on = false; + if (IsKeyword(request, 2, "on")) on = true; + else if (IsKeyword(request, 2, "off")) on = false; else return TypedRedisValue.Error("ERR syntax error"); string movingEndpointType = null; @@ -743,9 +787,9 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i { if (i + 1 >= request.Count) return TypedRedisValue.Error("ERR syntax error"); - if (request.IsString(i, "moving-endpoint-type"u8)) + if (IsKeyword(request, i, "moving-endpoint-type")) { - movingEndpointType = request.GetString(i + 1); + movingEndpointType = request.GetString(i + 1)?.ToLowerInvariant(); switch (movingEndpointType) { case "internal-ip": @@ -767,6 +811,11 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i client.MaintenanceNotifications = on; client.MovingEndpointType = on ? movingEndpointType : null; client.MaintenanceNotificationOptInCount++; + if (on) + { + OnMaintenanceOptIn(); + ReplayRetainedCompletion(client); // the catch-up channel, immediately after this +OK + } Log($"[{client}] maintenance notifications {(on ? "on" : "off")}, moving-endpoint-type: {movingEndpointType ?? "(server default)"}"); return TypedRedisValue.OK; } @@ -1080,6 +1129,12 @@ public Node(RedisServer server, EndPoint endpoint, NodeFlags flags) } public void UpdateSlots(SlotRange[] slots) => _slots = slots; + + /// + /// Whether this node serves anything at all; note a null slot set means "all slots" + /// (the single-node default), not "none". + /// + public bool HasAnySlot => _slots is null || _slots.Length != 0; public ReadOnlySpan Slots => _slots ?? SlotRange.SharedAllSlots; public bool CheckCrossSlot => _server.CheckCrossSlot; From 3d9b3a6cf28e5b48800f3a9746045a43318b4996 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 15:00:08 +0100 Subject: [PATCH 03/36] D8: document the notifications, and make the diagnostic it recommends work (#3204) docs/ServerMaintenanceEvent.md already existed for the Azure pub/sub family, so this extends it rather than adding a page: the intro now distinguishes the two routes, and a new section covers the server-native RESP3 family. The section leads on the thing most likely to bite a user: for a recognised hostname the feature is automatic, because the matching options provider turns it on - but a custom domain, a CNAME, private DNS, a proxy, or a self-managed cluster matches nothing, so it falls back to Disabled and *nothing fails*. You simply never get notifications. Both fixes are spelled out, with their differing blast radius: defaults= for the whole posture, or maintNotifications= Auto for this feature alone. Related: RESP3 needs no configuration (with no protocol set the client assumes 6.0 and negotiates it), but three settings silently take it away - protocol= resp2, defaultVersion below 6.0, and disabling or renaming HELLO - and without RESP3 there are no push frames to receive. Writing the "how do I check it is on?" section turned up that the diagnostic did not exist. The opt-in *refusal* was reported through PhysicalConnection.OnDetailLog, which is [Conditional("PARSE_DETAIL")] and compiles away in any normal build, so the reason a server declined was visible only to somebody debugging the parser; acceptance was not reported at all. All three - accepted, refused, and the handoff outcome - now go through the configured ILoggerFactory as LoggerMessage extensions (event ids 117-119), which is the channel that survives and the one an application actually reads. The handoff line closes a gap noted earlier: Multiplexer.Trace is [Conditional("VERBOSE")], so a handoff that replaced connections left no record. Two tests assert the accepted and refused messages, since they are documented behaviour now rather than incidental logging. --- docs/ServerMaintenanceEvent.md | 152 +++++++++++++++++- docs/index.md | 2 +- src/StackExchange.Redis/LoggerExtensions.cs | 18 +++ src/StackExchange.Redis/ResultProcessor.cs | 2 +- .../ServerEndPoint.Maintenance.cs | 25 ++- .../MaintenanceOptInClientTests.cs | 56 ++++++- 6 files changed, 248 insertions(+), 7 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 2f4ba1c29..83de1563b 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -1,7 +1,14 @@ -# Introducing ServerMaintenanceEvents +# Introducing ServerMaintenanceEvents StackExchange.Redis now automatically subscribes to notifications about upcoming maintenance from supported Redis providers. The ServerMaintenanceEvent on the ConnectionMultiplexer raises events in response to notifications about server maintenance, and application code can subscribe to the event to handle connection drops more gracefully during these maintenance operations. +There are two sources of these events, and they arrive by completely different routes: + +* **Azure Cache for Redis** publishes them on a pub/sub channel (`AzureRedisEvents`), and they surface as `AzureMaintenanceEvent`. This is the original support, and is described below. +* **Redis Enterprise and Redis Cloud** send them as RESP3 *push frames* on the connection that carries your commands, and they surface as `PushMaintenanceEvent`. This is newer, does more than report, and is covered in [its own section](#server-native-maintenance-notifications-redis-enterprise-and-redis-cloud). + +Both raise the same `ServerMaintenanceEvent` event, so a handler can watch for either. + If you are a Redis vendor and want to integrate support for ServerMaintenanceEvents into StackExchange.Redis, we recommend opening an issue so we can discuss the details. ## Types of events @@ -64,4 +71,145 @@ It's important to understand that this does *not* mean downtime if you are using #### NodeMaintenanceEnded event -`NodeMaintenanceEnded` events are raised to indicate that the maintenance operation has completed and that the replica is once again available. You do *NOT* need to wait for this event to use the load balancer endpoint, as it is available throughout. However, we included this for logging purposes and for customers who use the replica endpoint in clusters for read workloads. \ No newline at end of file +`NodeMaintenanceEnded` events are raised to indicate that the maintenance operation has completed and that the replica is once again available. You do *NOT* need to wait for this event to use the load balancer endpoint, as it is available throughout. However, we included this for logging purposes and for customers who use the replica endpoint in clusters for read workloads. + +# Server-native maintenance notifications (Redis Enterprise and Redis Cloud) + +> These APIs are experimental, behind diagnostic id `SER010`; see [SER010](exp/SER010.md). + +Redis Enterprise and Redis Cloud can tell a client *directly* that a disruption is coming: a shard is migrating, a node is failing over, or the endpoint you are connected to is being replaced. Unlike the Azure events above, these arrive as RESP3 push frames on the connection itself, and the client does not merely report them: it relaxes timeouts for the duration, re-reads the cluster topology when slots have moved, recovers sharded subscriptions that were stranded, and moves off an endpoint that is going away rather than waiting to be disconnected. + +## Do I need to configure anything? + +Usually not. If you connect using the hostname your provider gave you, the matching options provider recognizes it and turns the feature on for you. + +| You connect to | Recognized as | Notifications | +|---|---|---| +| `something.cloud.redislabs.com`, `.cloud.redis.io`, `.redislabs.com` | Redis Cloud | on (`Auto`) | +| `something.redis.azure.net`, `.redisenterprise.cache.azure.net` | Azure Managed Redis | on (`Auto`) | +| your own hostname, a CNAME, private DNS, or through a proxy | nothing | **off** | +| a self-managed Redis Enterprise cluster | nothing (there is no DNS pattern to recognize) | **off** | + +The last two rows are the ones to know about, because nothing fails: the connection works normally and you simply never receive a notification. If your endpoint does not look like your provider's, say so explicitly. Either: + +```csharp +// the whole deployment posture: prefer RESP3, skip the OSS config-broadcast channel, and ask for notifications +var options = ConfigurationOptions.Parse("my-redis.internal.example.com:6379,defaults=enterprise"); +``` + +or, to change nothing except this feature: + +```csharp +var options = ConfigurationOptions.Parse("my-redis.internal.example.com:6379,maintNotifications=Auto"); +``` + +`defaults=` accepts `rediscloud`, `enterprise`, `amr` and `azure`; see [Configuration](Configuration.md) for what each provider sets. It is also the right answer for a *hosted* deployment reached somewhere its own provider cannot see it, such as behind a CNAME or a private endpoint. + +### RESP3 is required, and is already the default + +These notifications are RESP3 push frames, so RESP3 is a hard requirement. You do not normally need to ask for it: with no protocol configured the client assumes a 6.0 server and negotiates RESP3, which is enough. But three settings take RESP3 away again, and each one silently disables this feature: + +* `protocol=resp2` (or `Protocol = RedisProtocol.Resp2`) +* `defaultVersion` below 6.0, which is how the client decides RESP3 is available at all +* disabling or renaming `HELLO` in the [command map](Configuration.md), since RESP3 is negotiated by `HELLO` + +If you use `maintNotifications=Enabled` (see below) you will find out about this immediately, because the connection will be refused rather than quietly running without the feature. + +## Choosing a mode + +```csharp +options.MaintenanceNotifications = MaintenanceNotificationMode.Auto; +``` + +| Mode | Meaning | +|---|---| +| `Disabled` | never ask. The default when nothing recognizes your endpoint | +| `Auto` | ask, and carry on if the server says no. What the providers select | +| `Enabled` | **require** them: if the server will not deliver them, or the connection ends up on RESP2, the connection is **rejected** | + +`Auto` is the right choice almost always: asking costs one command during the handshake, and a server that accepts and then never sends anything costs nothing at all. `Enabled` exists for the case where running without advance warning is worse than not running: it turns a silent absence into a startup failure, which also makes it a useful way to prove the feature is live in a staging environment. + +## What the client does without your involvement + +| Notification | What the client does | +|---|---| +| `MIGRATING`, `FAILING_OVER`, `SMIGRATING` | relaxes command timeouts for that server while the disruption lasts | +| `MIGRATED`, `FAILED_OVER` | ends the window, but keeps timeouts relaxed for a short tail while things settle | +| `SMIGRATED` | as above, and re-reads the cluster topology, and re-subscribes any sharded channels whose slots moved | +| `MOVING` | works out the replacement address, lets in-flight work finish, then replaces the connections | + +So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. + +Note that a deliberate handoff appears as a `ConnectionFailed` event with `FailureType == ConnectionFailureType.MaintenanceHandoff`. That is expected during planned maintenance and does not indicate a fault; if you alert on `ConnectionFailed`, filter it out. + +## Watching the events + +```csharp +multiplexer.ServerMaintenanceEvent += (sender, e) => +{ + if (e is PushMaintenanceEvent maintenance) + { + logger.LogInformation( + "{Type} from {EndPoint} (seq {Sequence}, {Time})", + maintenance.NotificationType, maintenance.EndPoint, maintenance.SequenceId, maintenance.Time); + + foreach (var migration in maintenance.SlotMigrations) + { + logger.LogInformation("slots {Slots}: {Source} -> {Target}", migration.RawSlots, migration.Source, migration.Target); + } + } +}; +``` + +Two things are worth knowing before you build on the detail: + +* **`EndPoint` is whichever node told us first.** Every node broadcasts a given event, so the client collapses the copies and raises one event; it is not necessarily the node being maintained, and for the cluster notifications it is usually a bystander reporting somebody else's movements. +* **`SequenceId` is observed behaviour, not a contract.** No specification defines it. In practice it is monotonic per database and shared across notification types, which makes it useful for spotting a replay, but do not depend on it across deployments or versions. + +`Time` is what the server announced, and may legitimately be zero or negative for a connection that arrived mid-window, meaning "this is happening now". + +## Timeouts during maintenance + +Three settings control the relaxed window, all in seconds: + +| Setting | Default | Meaning | +|---|---|---| +| `maintRelaxedTimeout` | 10s | the timeout to use while a disruption is in progress, and the floor for how long a window lasts | +| `maintRelaxedWindowMax` | 3x the relaxed timeout | the longest a single window may last, in case a closing notification never arrives | +| `maintPostEventRelaxed` | 2x the relaxed timeout | how long timeouts stay relaxed *after* the disruption ends | + +The announced duration is clamped rather than honoured literally. Windows as short as two seconds have been observed in practice, which is not long enough to cover a client reconnecting, and a client that trusted the announced value would stop being patient exactly when it mattered. The tail exists for the same reason in reverse: after a handoff, servers and other clients are still settling. + +If a command does time out during a window, the exception carries the reason: `RedisTimeoutException.MaintenanceType` (and the same property on `RedisConnectionException`) names the notification that was in effect, which distinguishes "the deployment was moving" from "this query is slow". + +## Checking that it is working + +Wire up an `ILoggerFactory` and the client reports the outcome of the opt-in, per server: + +```csharp +options.LoggerFactory = loggerFactory; +await using var multiplexer = await ConnectionMultiplexer.ConnectAsync(options); +``` + +``` +10.0.0.1:6379: Requesting maintenance notifications (Auto) +10.0.0.1:6379: Maintenance notifications accepted +``` + +or, when the server declines, the reason it gave: + +``` +10.0.0.1:6379: Maintenance notifications refused (ERR maintenance notifications are disabled on this server) +``` + +A handoff is reported the same way, which is worth knowing because it replaces connections: + +``` +10.0.0.1:6379: Maintenance handoff: Recycle -> 10.0.0.2:6379: db.example.com now resolves to 10.0.0.2:6379 +``` + +Alternatively set `MaintenanceNotifications = Enabled` in a test or staging environment: if anything prevents the feature working, including ending up on RESP2, the connection fails instead of running silently without it. + +## Which deployments send these + +Redis Enterprise and Redis Cloud send them, subject to the feature being enabled on the cluster. Azure Managed Redis is configured to ask for them ahead of its own rollout, so the setting is harmless until their servers begin emitting. Redis Open Source, Valkey and other servers do not send them at all, and the setting is simply inert there: the opt-in is refused and the client carries on. diff --git a/docs/index.md b/docs/index.md index 93c6eab7d..0665045ed 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,7 +45,7 @@ Documentation - [Pub/Sub Key Notifications](KeyspaceNotifications) - how to use keyspace and keyevent notifications - [Hot Keys](HotKeys) - how to use `HOTKEYS` profiling - [Using RESP3](Resp3) - information on using RESP3 -- [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance (e.g. Azure Cache for Redis) +- [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance, including the server-native notifications sent by Redis Enterprise and Redis Cloud - [Streams](Streams) - how to use the Stream data type - [Arrays](Arrays) - how to use Redis Arrays as sparse arrays of values - [Vector Sets](VectorSets) - how to use Vector Sets for similarity search with embeddings diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 7febd6dde..27026c28e 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -771,4 +771,22 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) EventId = 116, Message = "{Server}: Requesting maintenance notifications ({Mode})")] internal static partial void LogInformationRequestingMaintenanceNotifications(this ILogger logger, ServerEndPointLogValue server, MaintenanceNotificationMode mode); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 117, + Message = "{Server}: Maintenance notifications accepted")] + internal static partial void LogInformationMaintenanceNotificationsAccepted(this ILogger logger, ServerEndPointLogValue server); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 118, + Message = "{Server}: Maintenance notifications refused ({Reason})")] + internal static partial void LogInformationMaintenanceNotificationsRefused(this ILogger logger, ServerEndPointLogValue server, string reason); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 119, + Message = "{Server}: Maintenance handoff: {Outcome}")] + internal static partial void LogInformationMaintenanceHandoff(this ILogger logger, ServerEndPointLogValue server, string outcome); } diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index f8eb06a0e..b8617de3d 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -3214,7 +3214,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r if (reader.IsScalar && Literals.OK.Hash.IsCS(reader.TryGetSpan(out var span) ? span : reader.Buffer(stackalloc byte[16]))) { - server?.OnMaintenanceNotificationsAccepted(); + server?.OnMaintenanceNotificationsAccepted(connection); SetResult(message, true); return true; } diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 28f00e1b0..55c928333 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -46,7 +46,19 @@ private bool ShouldRequestMaintenanceNotifications(bool isInteractive, bool nego && MaintenanceMode != MaintenanceNotificationMode.Disabled && Multiplexer.CommandMap.IsAvailable(RedisCommand.CLIENT); - internal void OnMaintenanceNotificationsAccepted() => _maintenanceNotificationsActive = true; + /// + /// The server agreed to send them. + /// + /// + /// Logged as well as recorded, so that the connect log answers "is this actually on?" outright. Previously + /// only the *refusal* was logged, which meant a working feature left no trace and could only be inferred + /// from the absence of a complaint - and that is indistinguishable from never having asked. + /// + internal void OnMaintenanceNotificationsAccepted(PhysicalConnection connection) + { + _maintenanceNotificationsActive = true; + Multiplexer.Logger?.LogInformationMaintenanceNotificationsAccepted(new(this)); + } /// /// The server declined our request. Recorded rather than acted on: whether that matters is a question for @@ -56,7 +68,11 @@ internal void OnMaintenanceNotificationsRefused(PhysicalConnection connection, s { _maintenanceNotificationsActive = false; _maintenanceNotificationsRefusal = reason; - connection.OnDetailLog($"maintenance notifications refused: {reason}"); + + // via the configured logger, not OnDetailLog: that is [Conditional("PARSE_DETAIL")] and compiles away + // in any normal build, so for as long as it was the only report of a refusal, the reason a server + // declined was invisible to everybody who was not debugging the parser. + Multiplexer.Logger?.LogInformationMaintenanceNotificationsRefused(new(this), reason); } /// @@ -281,6 +297,11 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? Multiplexer.Trace($"MOVING: {decision}", ToString()); _lastHandoffOutcome = decision.ToString(); + + // Trace is [Conditional("VERBOSE")], so without this a handoff leaves no record in a normal build - + // and a handoff replaces connections, which is exactly the kind of thing somebody needs to be able + // to find afterwards. + Multiplexer.Logger?.LogInformationMaintenanceHandoff(new(this), _lastHandoffOutcome); switch (decision.Action) { case HandoffAction.Recycle: diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs index 7da72d32d..eaea0f550 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -1,6 +1,8 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Xunit; using static StackExchange.Redis.Server.RedisServer; @@ -208,6 +210,58 @@ public async Task AutoIsHappyOnResp2() Assert.Equal("value", await Set(conn)); } + /// + /// Captures log messages so a test can assert on what an operator would actually see. + /// + private sealed class CapturingLoggerFactory : ILoggerFactory, ILogger + { + public List Messages { get; } = []; + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (Messages) Messages.Add(formatter(state, exception)); + } + + public string All + { + get { lock (Messages) return string.Join("\n", Messages); } + } + + public void Dispose() { } + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.Supported, "Maintenance notifications accepted")] + [InlineData(MaintenanceNotificationSupport.Disabled, "Maintenance notifications refused")] + public async Task TheLogSaysWhetherTheFeatureIsLive(MaintenanceNotificationSupport support, string expected) + { + // This is the diagnostic docs/ServerMaintenanceEvent.md tells people to use, so it is worth a test. + // It also guards a mistake that was live for a while: the refusal was reported via + // PhysicalConnection.OnDetailLog, which is [Conditional("PARSE_DETAIL")] and compiles away in any normal + // build - so the reason a server declined was invisible to everybody who was not debugging the parser. + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + + var captured = new CapturingLoggerFactory(); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.LoggerFactory = captured; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + log.WriteLine(captured.All); + Assert.Contains(expected, captured.All); + } + [Fact] public async Task OptInIsReArmedOnReconnect() { From b44132a5c6bd4cef77ada9f0877ebaa1310aec2e Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 15:41:12 +0100 Subject: [PATCH 04/36] Ask for a moving-endpoint-type, and find the successor field was never empty Adds MaintenanceEndpointType, ConfigurationOptions.MaintenanceMovingEndpointType and the maintMovingEndpointType key, sent as moving-endpoint-type on the opt-in whenever it is anything but ServerDefault. The reason it matters is the experiment it enabled. Eleven MOVING notifications had been observed carrying an explicit null, and the notes had concluded the build did not populate the field. That was wrong: every one of those was requested with a bare CLIENT MAINT_NOTIFICATIONS ON, and a bare ON means "server defaults", which amounts to none. Asking explicitly produces an address every time - measured across all four forms on RS 8.0.22: external-ip 34.253.226.6:13796 internal-ip 10.0.101.58:13377 external-fqdn node2.:13216 internal-fqdn node2.internal.:13049 All four parse, the FQDN forms as DnsEndPoint. ServerDefault remains the default so nothing changes silently; the auto derivation the contract prescribes - private-versus-public choosing the scope, TLS choosing ip-versus-fqdn because a certificate cannot be validated against a bare address - is still to come, and should then become the default. Also fixes an assertion that had baked in the old belief: the live handoff test required the outcome to be a "Recycle", but with a named successor it is correctly a "Reconfigure". Tests: the parameter is sent for each type and omitted for ServerDefault, an unsupported type is refused without failing the connection (the fake's accepted set is now configurable), and a live experiment records what the server returns per type. Three guard tests updated deliberately for the new field, key and literals. --- .../Configuration/DefaultOptionsProvider.cs | 11 + .../ConfigurationOptions.cs | 36 +++ .../MaintenanceEndpointType.cs | 52 ++++ .../PublicAPI/PublicAPI.Unshipped.txt | 234 +++++++++--------- src/StackExchange.Redis/RedisLiterals.cs | 5 + .../ServerEndPoint.Maintenance.cs | 13 + src/StackExchange.Redis/ServerEndPoint.cs | 8 +- .../MovingEndpointTypeScenarioTests.cs | 114 +++++++++ .../MovingHandoffScenarioTests.cs | 21 +- .../StackExchange.Redis.Tests/ConfigTests.cs | 1 + .../MaintenanceOptInClientTests.cs | 44 ++++ .../RedisValueStorageKindUnitTests.cs | 2 +- .../StackExchange.Redis.Server/RedisServer.cs | 23 +- 13 files changed, 436 insertions(+), 128 deletions(-) create mode 100644 src/StackExchange.Redis/MaintenanceEndpointType.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index a278f435f..8788892a1 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -350,6 +350,17 @@ protected virtual string GetDefaultClientName() => [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] public virtual MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Disabled; + /// + /// Which form of address to ask a server to name when an endpoint moves. + /// + /// + /// - ask for nothing, and let the server decide. + /// A provider that knows how its deployment is reached can do better: an FQDN form is the right answer + /// wherever TLS is in play, because an address cannot be validated against a DNS-only certificate. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual MaintenanceEndpointType MaintenanceMovingEndpointType => MaintenanceEndpointType.ServerDefault; + /// /// Gets the value command timeouts are relaxed to during an announced disruption; 10 seconds, as the /// notification contract prescribes. diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 94b95f7c0..0ef37e53e 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -91,6 +91,15 @@ internal static MaintenanceNotificationMode ParseMaintenanceNotifications(string return tmp; } + internal static MaintenanceEndpointType ParseMaintenanceEndpointType(string key, string value) + { + if (!Enum.TryParse(value, true, out MaintenanceEndpointType tmp) || !Enum.IsDefined(typeof(MaintenanceEndpointType), tmp)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a MaintenanceEndpointType value; the value '{value}' is not recognised."); + } + return tmp; + } + /// /// Parses one of the maintenance durations, which are expressed in seconds - the unit the /// cross-client contract uses for maintRelaxedTimeout, so a documented value can be pasted @@ -170,6 +179,7 @@ internal const string Protocol = "protocol", Defaults = "defaults", MaintenanceNotifications = "maintNotifications", + MaintenanceMovingEndpointType = "maintMovingEndpointType", MaintenanceRelaxedTimeout = "maintRelaxedTimeout", MaintenanceRelaxedWindowMax = "maintRelaxedWindowMax", MaintenancePostEventRelaxedDuration = "maintPostEventRelaxed", @@ -211,6 +221,7 @@ internal const string SetClientLibrary, Protocol, Defaults, + MaintenanceMovingEndpointType, MaintenanceNotifications, MaintenanceRelaxedTimeout, MaintenanceRelaxedWindowMax, @@ -271,6 +282,7 @@ private enum OptionFlags : ulong ProtocolHasValue = 1UL << 33, AllowSimulateConnectionFailure = 1UL << 34, MaintenanceNotificationsHasValue = 1UL << 35, + MaintenanceMovingEndpointTypeHasValue = 1UL << 40, DefaultsHasValue = 1UL << 36, MaintenanceRelaxedTimeoutHasValue = 1UL << 37, MaintenanceRelaxedWindowMaxHasValue = 1UL << 38, @@ -301,6 +313,7 @@ private enum OptionFlags : ulong private RedisProtocol _protocol; private MaintenanceNotificationMode _maintenanceNotifications; + private MaintenanceEndpointType _maintenanceMovingEndpointType; private TimeSpan _maintenanceRelaxedTimeout, _maintenanceRelaxedWindowMax, _maintenancePostEventRelaxedDuration; private bool HasValue(OptionFlags hasValue) => (optionFlags & hasValue) != 0; @@ -1173,6 +1186,7 @@ public string ToString(bool includePassword) // the string, or re-parsing would pin a choice that was only ever a guess from the endpoints if (HasValue(OptionFlags.DefaultsHasValue) && defaultOptions?.Name is { } defaultsName) Append(sb, OptionKeys.Defaults, defaultsName); if (HasValue(OptionFlags.MaintenanceNotificationsHasValue)) Append(sb, OptionKeys.MaintenanceNotifications, _maintenanceNotifications.ToString()); + if (HasValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue)) Append(sb, OptionKeys.MaintenanceMovingEndpointType, _maintenanceMovingEndpointType.ToString()); if (HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedTimeout, FormatMaintenanceSeconds(_maintenanceRelaxedTimeout)); if (HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedWindowMax, FormatMaintenanceSeconds(_maintenanceRelaxedWindowMax)); if (HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue)) Append(sb, OptionKeys.MaintenancePostEventRelaxedDuration, FormatMaintenanceSeconds(_maintenancePostEventRelaxedDuration)); @@ -1455,6 +1469,9 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) case OptionKeys.MaintenanceNotifications: SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, OptionKeys.ParseMaintenanceNotifications(key, value)); break; + case OptionKeys.MaintenanceMovingEndpointType: + SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, OptionKeys.ParseMaintenanceEndpointType(key, value)); + break; case OptionKeys.MaintenanceRelaxedTimeout: SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, OptionKeys.ParseMaintenanceSeconds(key, value)); break; @@ -1534,6 +1551,25 @@ public MaintenanceNotificationMode MaintenanceNotifications set => SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, value); } + /// + /// Which form of address a server should name when it announces that an endpoint is moving. + /// + /// + /// Sent as moving-endpoint-type on the maintenance-notification opt-in. + /// (the default) sends no preference at all, which + /// is what the client has always done - and every MOVING observed that way carried no address, so + /// a client that wants a named replacement should ask for one. Prefer an FQDN form under TLS: an address + /// cannot be validated against a certificate carrying only DNS names. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceEndpointType MaintenanceMovingEndpointType + { + get => HasValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue) + ? _maintenanceMovingEndpointType + : Defaults.MaintenanceMovingEndpointType; + set => SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, value); + } + /// /// The value command timeouts are relaxed to while a server has announced a disruption. /// diff --git a/src/StackExchange.Redis/MaintenanceEndpointType.cs b/src/StackExchange.Redis/MaintenanceEndpointType.cs new file mode 100644 index 000000000..0711bfd87 --- /dev/null +++ b/src/StackExchange.Redis/MaintenanceEndpointType.cs @@ -0,0 +1,52 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis; + +/// +/// Which form of address a server should name when it tells us an endpoint is moving. +/// +/// +/// Sent as the moving-endpoint-type parameter of the maintenance-notification opt-in, and it decides what +/// arrives in . +/// +/// Worth asking for rather than leaving to the server. Eleven observed MOVING notifications on Redis +/// Enterprise 8.0.22 all carried an explicit null, including ones where the server had already chosen the +/// replacement node - and every one of those was requested with a bare ON, so the working theory is that +/// the server default amounts to and we were getting what we asked for. +/// +/// +/// The choice is not cosmetic where TLS is involved: a certificate that carries DNS names and no IP SAN cannot +/// validate an address, so a verifying client that is handed an IP cannot use it. Prefer the FQDN forms when +/// connecting with TLS. +/// +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceEndpointType +{ + /// + /// Do not ask; let the server choose. This is the default, and matches what the client has always sent. + /// + /// + /// In practice this has been observed to mean "no address at all", so a client that wants a named + /// replacement should ask for one explicitly. + /// + ServerDefault = 0, + + /// A private address, for a client inside the deployment's network. + InternalIp, + + /// A private hostname, for a client inside the deployment's network. + InternalFqdn, + + /// A public address. Note an address cannot be validated against a DNS-only certificate. + ExternalIp, + + /// A public hostname. The right choice when connecting with TLS. + ExternalFqdn, + + /// + /// Explicitly ask for no address, so a handoff always goes back through the endpoint as configured. + /// + None, +} diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 4b87978f5..36d4b6364 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,20 +1,19 @@ #nullable enable -StackExchange.Redis.ConfigurationOptions.SentinelPassword.get -> string? -StackExchange.Redis.ConfigurationOptions.SentinelPassword.set -> void -StackExchange.Redis.ConfigurationOptions.SentinelUser.get -> string? -StackExchange.Redis.ConfigurationOptions.SentinelUser.set -> void -[SER009]StackExchange.Redis.Configuration.TlsOptions -[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions() -> void -[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions(StackExchange.Redis.ConfigurationOptions! options) -> void -[SER009]StackExchange.Redis.Configuration.TlsOptions.IsEnabled.get -> bool -[SER009]StackExchange.Redis.Configuration.TlsOptions.SslHost.get -> string? -[SER009]StackExchange.Redis.Configuration.TlsOptions.SslProtocols.get -> System.Security.Authentication.SslProtocols? -[SER009]StackExchange.Redis.Configuration.TlsOptions.CheckCertificateRevocation.get -> bool -[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateValidationCallback.get -> System.Net.Security.RemoteCertificateValidationCallback? -[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateSelectionCallback.get -> System.Net.Security.LocalCertificateSelectionCallback? -[SER009]StackExchange.Redis.Configuration.TlsOptions.ResolveHost(System.Net.EndPoint! endpoint) -> string! -[SER009]virtual StackExchange.Redis.Configuration.Tunnel.ConnectTransportAsync(System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType, StackExchange.Redis.Configuration.TlsOptions tls, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask -StackExchange.Redis.RedisFeatures.Hello.get -> bool +StackExchange.Redis.BitFieldEncoding +StackExchange.Redis.BitFieldEncoding.BitFieldEncoding() -> void +StackExchange.Redis.BitFieldEncoding.Equals(StackExchange.Redis.BitFieldEncoding other) -> bool +StackExchange.Redis.BitFieldEncoding.IsSigned.get -> bool +StackExchange.Redis.BitFieldEncoding.Width.get -> int +StackExchange.Redis.BitFieldOffset +StackExchange.Redis.BitFieldOffset.BitFieldOffset() -> void +StackExchange.Redis.BitFieldOffset.Equals(StackExchange.Redis.BitFieldOffset other) -> bool +StackExchange.Redis.BitFieldOperation +StackExchange.Redis.BitFieldOperation.BitFieldOperation() -> void +StackExchange.Redis.BitFieldOperation.Equals(StackExchange.Redis.BitFieldOperation other) -> bool +StackExchange.Redis.BitFieldOverflow +StackExchange.Redis.BitFieldOverflow.Fail = 2 -> StackExchange.Redis.BitFieldOverflow +StackExchange.Redis.BitFieldOverflow.Saturate = 1 -> StackExchange.Redis.BitFieldOverflow +StackExchange.Redis.BitFieldOverflow.Wrap = 0 -> StackExchange.Redis.BitFieldOverflow StackExchange.Redis.ClusterNode.AuxFields.get -> System.Collections.Generic.IReadOnlyList>! StackExchange.Redis.ClusterNode.ClusterBusPort.get -> int? StackExchange.Redis.ClusterNode.Hostname.get -> string? @@ -32,14 +31,27 @@ StackExchange.Redis.ClusterSlotNode.NodeId.get -> string? StackExchange.Redis.ClusterSlotNode.Port.get -> int StackExchange.Redis.ClusterSlotsResult StackExchange.Redis.ClusterSlotsResult.Assignments.get -> System.Collections.Generic.IReadOnlyList! -StackExchange.Redis.IServer.ClusterSlots(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.ClusterSlotsResult? -StackExchange.Redis.IServer.ClusterSlotsAsync(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.RedisErrorKind.UnknownRedirectTarget = 26 -> StackExchange.Redis.RedisErrorKind -override StackExchange.Redis.ClusterSlotNode.ToString() -> string! -StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.NameValueEntry[]! streamPairs, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue streamField, StackExchange.Redis.RedisValue streamValue, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.Configuration.RedisCloudOptionsProvider +StackExchange.Redis.Configuration.RedisCloudOptionsProvider.RedisCloudOptionsProvider() -> void +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.RedisEnterpriseOptionsProvider() -> void +StackExchange.Redis.ConfigurationOptions.SentinelPassword.get -> string? +StackExchange.Redis.ConfigurationOptions.SentinelPassword.set -> void +StackExchange.Redis.ConfigurationOptions.SentinelUser.get -> string? +StackExchange.Redis.ConfigurationOptions.SentinelUser.set -> void +StackExchange.Redis.ConnectionFailureType.MaintenanceHandoff = 12 -> StackExchange.Redis.ConnectionFailureType StackExchange.Redis.IDatabase.StreamAdd(StackExchange.Redis.RedisKey key, StackExchange.Redis.NameValueEntry[]! streamPairs, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue StackExchange.Redis.IDatabase.StreamAdd(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue streamField, StackExchange.Redis.RedisValue streamValue, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue +StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long? +StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease! +StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.NameValueEntry[]! streamPairs, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue streamField, StackExchange.Redis.RedisValue streamValue, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task!>! +StackExchange.Redis.IServer.ClusterSlots(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.ClusterSlotsResult? +StackExchange.Redis.IServer.ClusterSlotsAsync(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.RedisFeatures.BitFieldReadOnly.get -> bool +StackExchange.Redis.RedisFeatures.Hello.get -> bool StackExchange.Redis.StreamAddOptions StackExchange.Redis.StreamAddOptions.Approximate.get -> bool StackExchange.Redis.StreamAddOptions.Approximate.init -> void @@ -59,6 +71,76 @@ StackExchange.Redis.StreamAddOptions.StreamAddOptions() -> void StackExchange.Redis.StreamAddOptions.TrimMode.get -> StackExchange.Redis.StreamTrimMode StackExchange.Redis.StreamAddOptions.TrimMode.init -> void StackExchange.Redis.StringIndex +[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeFlags.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.RedisErrorKind.UnknownRedirectTarget = 26 -> StackExchange.Redis.RedisErrorKind +[SER009]StackExchange.Redis.Configuration.TlsOptions +[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateSelectionCallback.get -> System.Net.Security.LocalCertificateSelectionCallback? +[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateValidationCallback.get -> System.Net.Security.RemoteCertificateValidationCallback? +[SER009]StackExchange.Redis.Configuration.TlsOptions.CheckCertificateRevocation.get -> bool +[SER009]StackExchange.Redis.Configuration.TlsOptions.IsEnabled.get -> bool +[SER009]StackExchange.Redis.Configuration.TlsOptions.ResolveHost(System.Net.EndPoint! endpoint) -> string! +[SER009]StackExchange.Redis.Configuration.TlsOptions.SslHost.get -> string? +[SER009]StackExchange.Redis.Configuration.TlsOptions.SslProtocols.get -> System.Security.Authentication.SslProtocols? +[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions() -> void +[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions(StackExchange.Redis.ConfigurationOptions! options) -> void +[SER009]virtual StackExchange.Redis.Configuration.Tunnel.ConnectTransportAsync(System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType, StackExchange.Redis.Configuration.TlsOptions tls, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +[SER010]StackExchange.Redis.Availability.FaultContext.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceMovingEndpointType.get -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceMovingEndpointType.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.set -> void +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.ClusterSlotMigration() -> void +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.RawSlots.get -> string? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Slots.get -> System.Collections.Generic.IReadOnlyList! +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Source.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Target.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailedOver = 5 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailingOver = 4 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrated = 3 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrating = 2 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Moving = 1 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.None = 0 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrated = 7 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrating = 6 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.EndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NewEndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NotificationType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Payload.get -> string? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SequenceId.get -> long +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SlotMigrations.get -> System.Collections.Generic.IReadOnlyList! +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Time.get -> System.TimeSpan? +[SER010]StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalFqdn = 4 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalIp = 3 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalFqdn = 2 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalIp = 1 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.None = 5 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ServerDefault = 0 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Auto = 2 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Disabled = 0 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Enabled = 1 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.RedisConnectionException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.RedisTimeoutException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Maintenance.ClusterSlotMigration.ToString() -> string! +[SER010]override StackExchange.Redis.Maintenance.PushMaintenanceEvent.ToString() -> string? +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceMovingEndpointType.get -> StackExchange.Redis.MaintenanceEndpointType +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan? +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedWindowMax.get -> System.TimeSpan? const StackExchange.Redis.StringIndex.Unbounded = -9223372036854775808 -> long override StackExchange.Redis.BitFieldEncoding.Equals(object? obj) -> bool override StackExchange.Redis.BitFieldEncoding.GetHashCode() -> int @@ -69,110 +151,38 @@ override StackExchange.Redis.BitFieldOffset.ToString() -> string! override StackExchange.Redis.BitFieldOperation.Equals(object? obj) -> bool override StackExchange.Redis.BitFieldOperation.GetHashCode() -> int override StackExchange.Redis.BitFieldOperation.ToString() -> string! -StackExchange.Redis.BitFieldEncoding -StackExchange.Redis.BitFieldEncoding.BitFieldEncoding() -> void -StackExchange.Redis.BitFieldEncoding.Equals(StackExchange.Redis.BitFieldEncoding other) -> bool -StackExchange.Redis.BitFieldEncoding.IsSigned.get -> bool -StackExchange.Redis.BitFieldEncoding.Width.get -> int -StackExchange.Redis.BitFieldOffset -StackExchange.Redis.BitFieldOffset.BitFieldOffset() -> void -StackExchange.Redis.BitFieldOffset.Equals(StackExchange.Redis.BitFieldOffset other) -> bool -StackExchange.Redis.BitFieldOperation -StackExchange.Redis.BitFieldOperation.BitFieldOperation() -> void -StackExchange.Redis.BitFieldOperation.Equals(StackExchange.Redis.BitFieldOperation other) -> bool -StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.BitFieldOverflow.Fail = 2 -> StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.BitFieldOverflow.Saturate = 1 -> StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.BitFieldOverflow.Wrap = 0 -> StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task!>! -StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long? -StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease! -StackExchange.Redis.RedisFeatures.BitFieldReadOnly.get -> bool +override StackExchange.Redis.ClusterSlotNode.ToString() -> string! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.AzureOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.DefaultOptionsProvider.ToString() -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.AbortOnConnectFail.get -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.IsMatch(System.Net.EndPoint! endpoint) -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? static StackExchange.Redis.BitFieldEncoding.Int16.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Int32.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Int64.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Int8.get -> StackExchange.Redis.BitFieldEncoding -static StackExchange.Redis.BitFieldEncoding.operator ==(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool -static StackExchange.Redis.BitFieldEncoding.operator !=(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool static StackExchange.Redis.BitFieldEncoding.Signed(int width) -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt16.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt32.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt63.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt8.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Unsigned(int width) -> StackExchange.Redis.BitFieldEncoding +static StackExchange.Redis.BitFieldEncoding.operator !=(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool +static StackExchange.Redis.BitFieldEncoding.operator ==(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool static StackExchange.Redis.BitFieldOffset.Bit(long bit) -> StackExchange.Redis.BitFieldOffset -static StackExchange.Redis.BitFieldOffset.implicit operator StackExchange.Redis.BitFieldOffset(long bit) -> StackExchange.Redis.BitFieldOffset static StackExchange.Redis.BitFieldOffset.Element(long element) -> StackExchange.Redis.BitFieldOffset -static StackExchange.Redis.BitFieldOffset.operator ==(StackExchange.Redis.BitFieldOffset x, StackExchange.Redis.BitFieldOffset y) -> bool +static StackExchange.Redis.BitFieldOffset.implicit operator StackExchange.Redis.BitFieldOffset(long bit) -> StackExchange.Redis.BitFieldOffset static StackExchange.Redis.BitFieldOffset.operator !=(StackExchange.Redis.BitFieldOffset x, StackExchange.Redis.BitFieldOffset y) -> bool +static StackExchange.Redis.BitFieldOffset.operator ==(StackExchange.Redis.BitFieldOffset x, StackExchange.Redis.BitFieldOffset y) -> bool static StackExchange.Redis.BitFieldOperation.Get(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset) -> StackExchange.Redis.BitFieldOperation static StackExchange.Redis.BitFieldOperation.IncrementBy(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset, long value, StackExchange.Redis.BitFieldOverflow overflow = StackExchange.Redis.BitFieldOverflow.Wrap) -> StackExchange.Redis.BitFieldOperation -static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool -static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.Set(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset, long value, StackExchange.Redis.BitFieldOverflow overflow = StackExchange.Redis.BitFieldOverflow.Wrap) -> StackExchange.Redis.BitFieldOperation -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.set -> void -[SER010]StackExchange.Redis.MaintenanceNotificationMode -[SER010]StackExchange.Redis.MaintenanceNotificationMode.Auto = 2 -> StackExchange.Redis.MaintenanceNotificationMode -[SER010]StackExchange.Redis.MaintenanceNotificationMode.Disabled = 0 -> StackExchange.Redis.MaintenanceNotificationMode -[SER010]StackExchange.Redis.MaintenanceNotificationMode.Enabled = 1 -> StackExchange.Redis.MaintenanceNotificationMode -[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode -[SER010]override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode -[SER010]override StackExchange.Redis.Maintenance.PushMaintenanceEvent.ToString() -> string? -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailedOver = 5 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailingOver = 4 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrated = 3 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrating = 2 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Moving = 1 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.None = 0 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrated = 7 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrating = 6 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.EndPoint.get -> System.Net.EndPoint? -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NewEndPoint.get -> System.Net.EndPoint? -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NotificationType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Payload.get -> string? -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SequenceId.get -> long -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Time.get -> System.TimeSpan? -override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.AbortOnConnectFail.get -> bool -override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.ConfigurationChannel.get -> string! -override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.IsMatch(System.Net.EndPoint! endpoint) -> bool -override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? -[SER010]override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode -StackExchange.Redis.Configuration.RedisCloudOptionsProvider -StackExchange.Redis.Configuration.RedisCloudOptionsProvider.RedisCloudOptionsProvider() -> void +static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool +static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.Name.get -> string? -override StackExchange.Redis.Configuration.AzureOptionsProvider.Name.get -> string! -override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.Name.get -> string! -override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Name.get -> string! -override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.ConfigurationChannel.get -> string! -override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Name.get -> string! -override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? -[SER010]override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode -StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider -StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.RedisEnterpriseOptionsProvider() -> void -override StackExchange.Redis.Configuration.DefaultOptionsProvider.ToString() -> string! -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.set -> void -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.get -> System.TimeSpan -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.set -> void -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.get -> System.TimeSpan -[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.set -> void -[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan? -[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedTimeout.get -> System.TimeSpan -[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedWindowMax.get -> System.TimeSpan? -[SER010]StackExchange.Redis.Availability.FaultContext.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.RedisConnectionException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]StackExchange.Redis.RedisTimeoutException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType -[SER010]override StackExchange.Redis.Maintenance.ClusterSlotMigration.ToString() -> string! -[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration -[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.ClusterSlotMigration() -> void -[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.RawSlots.get -> string? -[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Slots.get -> System.Collections.Generic.IReadOnlyList! -[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Source.get -> System.Net.EndPoint? -[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Target.get -> System.Net.EndPoint? -[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SlotMigrations.get -> System.Collections.Generic.IReadOnlyList! -[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeFlags.get -> StackExchange.Redis.CommandFlags -StackExchange.Redis.ConnectionFailureType.MaintenanceHandoff = 12 -> StackExchange.Redis.ConnectionFailureType diff --git a/src/StackExchange.Redis/RedisLiterals.cs b/src/StackExchange.Redis/RedisLiterals.cs index d66cb841f..58d4632c9 100644 --- a/src/StackExchange.Redis/RedisLiterals.cs +++ b/src/StackExchange.Redis/RedisLiterals.cs @@ -73,6 +73,11 @@ public static readonly RedisValue LIST = RedisValue.FromRaw("LIST"u8), LT = RedisValue.FromRaw("LT"u8), MAINT_NOTIFICATIONS = RedisValue.FromRaw("MAINT_NOTIFICATIONS"u8), + moving_endpoint_type = RedisValue.FromRaw("moving-endpoint-type"u8), + internal_ip = RedisValue.FromRaw("internal-ip"u8), + internal_fqdn = RedisValue.FromRaw("internal-fqdn"u8), + external_ip = RedisValue.FromRaw("external-ip"u8), + external_fqdn = RedisValue.FromRaw("external-fqdn"u8), MATCH = RedisValue.FromRaw("MATCH"u8), MALLOC_STATS = RedisValue.FromRaw("MALLOC-STATS"u8), MAX = RedisValue.FromRaw("MAX"u8), diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 55c928333..4823088db 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -54,6 +54,19 @@ private bool ShouldRequestMaintenanceNotifications(bool isInteractive, bool nego /// only the *refusal* was logged, which meant a working feature left no trace and could only be inferred /// from the absence of a complaint - and that is indistinguishable from never having asked. /// + /// + /// The wire value for the configured moving-endpoint-type, or null to send no preference. + /// + private RedisValue MaintenanceMovingEndpointTypeLiteral => Multiplexer.RawConfig.MaintenanceMovingEndpointType switch + { + MaintenanceEndpointType.InternalIp => RedisLiterals.internal_ip, + MaintenanceEndpointType.InternalFqdn => RedisLiterals.internal_fqdn, + MaintenanceEndpointType.ExternalIp => RedisLiterals.external_ip, + MaintenanceEndpointType.ExternalFqdn => RedisLiterals.external_fqdn, + MaintenanceEndpointType.None => RedisLiterals.none, + _ => RedisValue.Null, // ServerDefault: a bare ON, which is what we have always sent + }; + internal void OnMaintenanceNotificationsAccepted(PhysicalConnection connection) { _maintenanceNotificationsActive = true; diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 49c408e06..e61c40eab 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1370,7 +1370,13 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) // downgrade once the reply has been processed. A bare ON is explicitly valid: the server // then picks the endpoint type, which is what we want until we derive one ourselves. log?.LogInformationRequestingMaintenanceNotifications(new(this), MaintenanceMode); - msg = Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON); + + // A bare ON leaves the endpoint type to the server, and every MOVING observed that way + // carried no address at all - so when a caller asks for a specific form, say so. + var endpointType = MaintenanceMovingEndpointTypeLiteral; + msg = endpointType.IsNull + ? Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON) + : Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, [RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON, RedisLiterals.moving_endpoint_type, endpointType]); msg.SetInternalCall(); await WriteDirectOrQueueFireAndForgetAsync(connection, msg, ResultProcessor.MaintenanceNotifications).ForAwait(); } diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs new file mode 100644 index 000000000..4f318be14 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Does asking for a moving-endpoint-type make the server name a replacement? +/// +/// +/// The experiment behind an open question. Eleven MOVING notifications observed on Redis Enterprise +/// 8.0.22 all carried an explicit null for the replacement address, including cases where the server had +/// already chosen the replacement node - and every one of those was requested with a bare +/// CLIENT MAINT_NOTIFICATIONS ON. So either this build never populates the field, or the server default +/// amounts to none and we were being given exactly what we asked for. +/// +/// This distinguishes those two, which matters for more than tidiness: a named successor would let a handoff +/// skip the DNS wait entirely, and DNS has been measured trailing the notification by up to 18.7s against a +/// socket that closed at 15.7s. It also decides whether the named-successor code path is reachable at all, or +/// exists only because the contract mentions it. +/// +/// +/// Deliberately reports rather than asserts a populated address: "this build does not populate it" is a +/// legitimate answer, and the test's job is to record which answer we got, per endpoint type. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "moving-endpoint-type")] +public class MovingEndpointTypeScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData(MaintenanceEndpointType.ServerDefault)] + [InlineData(MaintenanceEndpointType.ExternalFqdn)] + [InlineData(MaintenanceEndpointType.ExternalIp)] + [InlineData(MaintenanceEndpointType.InternalFqdn)] + [InlineData(MaintenanceEndpointType.InternalIp)] + public async Task DoesTheServerNameAReplacement(MaintenanceEndpointType type) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", "conn_drop", "endpoint_rebind", log.WriteLine, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var config = database.GetClientConfig(fixture.Environment, MaintenanceNotificationMode.Auto); + config.MaintenanceMovingEndpointType = type; + + var clock = Stopwatch.StartNew(); + var moving = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + + // Auto rather than Enabled, because a server that rejects an endpoint type we asked for is one of the + // outcomes being measured - and it should not fail the run. + log.WriteLine($"requested {type}; opt-in active = {endpoint.MaintenanceNotificationsActive}"); + if (!endpoint.MaintenanceNotificationsActive) + { + log.WriteLine("=> the server refused this endpoint type; nothing further to observe"); + return; + } + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} {push.RawMessage}"); + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + lock (moving) moving.Add(push); + } + } + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + lock (moving) + { + if (moving.Count > 0) break; + } + + await Task.Delay(500, cancellationToken); + } + + lock (moving) + { + if (moving.Count == 0) + { + log.WriteLine("=> no MOVING arrived at all"); + return; + } + + foreach (var push in moving) + { + log.WriteLine($"=> {type}: NewEndPoint = {push.NewEndPoint?.ToString() ?? "(null)"}; payload = {push.Payload ?? "(null)"}"); + } + + // The one thing worth asserting either way: whatever the server sent, we understood the frame. + Assert.All(moving, push => Assert.Equal(MaintenanceNotificationType.Moving, push.NotificationType)); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs index 14fe17dbd..556e33618 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs @@ -28,9 +28,10 @@ public class MovingHandoffScenarioTests(ExistingDatabaseFixture fixture, ITestOu : IClassFixture { [Theory] - [InlineData("conn_drop", "endpoint_rebind")] - [InlineData("data_movement_conn_drop", "maintenance_mode")] - public async Task HandoffBeatsTheServerToTheClose(string effect, string trigger) + [InlineData("conn_drop", "endpoint_rebind", MaintenanceEndpointType.ServerDefault)] + [InlineData("data_movement_conn_drop", "maintenance_mode", MaintenanceEndpointType.ServerDefault)] + [InlineData("conn_drop", "endpoint_rebind", MaintenanceEndpointType.ExternalFqdn)] + public async Task HandoffBeatsTheServerToTheClose(string effect, string trigger, MaintenanceEndpointType endpointType) { fixture.RequireAvailable(); var cancellationToken = TestContext.Current.CancellationToken; @@ -51,7 +52,11 @@ void Note(string what) log.WriteLine(entry); } - await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var config = database.GetClientConfig(fixture.Environment); + config.MaintenanceMovingEndpointType = endpointType; + log.WriteLine($"moving-endpoint-type: {endpointType}"); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); var muxer = (IInternalConnectionMultiplexer)conn; var endpoint = muxer.GetServerEndPoint(conn.GetEndPoints()[0]); @@ -110,7 +115,13 @@ void Note(string what) // that would catch a regression. Assert.Equal(1, endpoint.HandoffRecycles); Assert.NotNull(endpoint.LastHandoffOutcome); - Assert.Contains("Recycle", endpoint.LastHandoffOutcome); + + // Recycle *or* Reconfigure: which one depends on whether the server named a replacement, and it only + // does that when we asked for an endpoint type. Asserting "Recycle" alone was an assumption from the + // era when the field was always null. + Assert.True( + endpoint.LastHandoffOutcome.Contains("Recycle") || endpoint.LastHandoffOutcome.Contains("Reconfigure"), + $"unexpected handoff outcome: {endpoint.LastHandoffOutcome}"); Assert.True( await Poll.UntilAsync( diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 06876e946..4a1a3dba9 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -62,6 +62,7 @@ orderby name Assert.Equal( new[] { + "_maintenanceMovingEndpointType", "_maintenanceNotifications", "_maintenancePostEventRelaxedDuration", "_maintenanceRelaxedTimeout", diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs index eaea0f550..7a781de59 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -262,6 +262,50 @@ public async Task TheLogSaysWhetherTheFeatureIsLive(MaintenanceNotificationSuppo Assert.Contains(expected, captured.All); } + [Theory] + [InlineData(MaintenanceEndpointType.ServerDefault, null)] + [InlineData(MaintenanceEndpointType.InternalIp, "internal-ip")] + [InlineData(MaintenanceEndpointType.InternalFqdn, "internal-fqdn")] + [InlineData(MaintenanceEndpointType.ExternalIp, "external-ip")] + [InlineData(MaintenanceEndpointType.ExternalFqdn, "external-fqdn")] + [InlineData(MaintenanceEndpointType.None, "none")] + public async Task MovingEndpointTypeIsSentWhenAskedFor(MaintenanceEndpointType type, string? expected) + { + // The point of asking: every MOVING observed on a real deployment carried no address, and every one of + // those was requested with a bare ON - so the working theory is that the server default amounts to + // "none". ServerDefault keeps that behaviour (send nothing); anything else says so explicitly. + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.MaintenanceMovingEndpointType = type; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(IsActive(conn, server), "the server should have accepted the opt-in"); + + var client = Assert.Single(OptedIn(server)); + log.WriteLine($"{type} -> moving-endpoint-type: {client.MovingEndpointType ?? "(none sent)"}"); + Assert.Equal(expected, client.MovingEndpointType); + } + + [Fact] + public async Task UnsupportedMovingEndpointTypeIsRefusedNotFatal() + { + // A server that does not know a type answers with an error, and that is a refusal like any other: with + // Auto we carry on without the feature rather than failing the connection. + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.SupportedMovingEndpointTypes = ["external-fqdn"]; // this deployment offers one form only + + var config = Config(server, MaintenanceNotificationMode.Auto); + config.MaintenanceMovingEndpointType = MaintenanceEndpointType.InternalIp; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.False(IsActive(conn, server), "an unsupported endpoint type is a refusal"); + Assert.Equal("value", await Set(conn)); // ...and the connection is still perfectly usable + } + [Fact] public async Task OptInIsReArmedOnReconnect() { diff --git a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs index 83ded0d7b..f23efe5fe 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs @@ -107,6 +107,6 @@ public void LiteralsResolveToExpectedStorageKinds() // only two storage flavors should ever appear; a third means something new crept in (e.g. a numeric // literal => Int64), which we want to notice Assert.Equal(2, byKind.Count); - Assert.Equal(23, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes + Assert.Equal(28, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes } } diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 88d20a262..a319ea303 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -755,6 +755,18 @@ public enum MaintenanceNotificationSupport /// public MaintenanceNotificationSupport MaintenanceNotifications { get; set; } + /// + /// The moving-endpoint-type values this server accepts. + /// + /// + /// Configurable because a deployment need not offer all of them - an internal address is meaningless to + /// a client outside the network, and a deployment with no public DNS cannot offer an external FQDN. A + /// client that asks for one it cannot have gets an error, and that is an ordinary refusal: the feature + /// stays off and the connection carries on. + /// + public string[] SupportedMovingEndpointTypes { get; set; } = + ["internal-ip", "internal-fqdn", "external-ip", "external-fqdn", "none"]; + // CLIENT MAINT_NOTIFICATIONS [parameter value ...], where parameter names follow a // $type-$setting convention and moving-endpoint-type is the only one defined so far. A bare ON is // explicitly valid and means "use the server defaults", so the parameter list is optional and @@ -790,16 +802,9 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i if (IsKeyword(request, i, "moving-endpoint-type")) { movingEndpointType = request.GetString(i + 1)?.ToLowerInvariant(); - switch (movingEndpointType) + if (movingEndpointType is null || Array.IndexOf(SupportedMovingEndpointTypes, movingEndpointType) < 0) { - case "internal-ip": - case "internal-fqdn": - case "external-ip": - case "external-fqdn": - case "none": - break; - default: - return TypedRedisValue.Error($"ERR unsupported moving-endpoint-type '{movingEndpointType}'"); + return TypedRedisValue.Error($"ERR unsupported moving-endpoint-type '{movingEndpointType}'"); } } else From 901f0674a07e070def90125cb3ce313f59887d77 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 15:56:30 +0100 Subject: [PATCH 05/36] Derive moving-endpoint-type per connection Adds MaintenanceEndpointType.Auto and the resolver behind it. Two independent questions, per the contract: scope comes from the address we actually reached, form comes from whether the connection is encrypted - TLS implies the FQDN variants, because a certificate generally cannot be validated against a bare address, so a client handed an IP mid-handoff could not verify where it was told to go. | private/reserved | otherwise TLS off | internal-ip | external-ip TLS on | internal-fqdn | external-fqdn Classifying the *connected* address rather than the configured endpoint matters: the latter is usually a name, and where it resolved to is what decides whether we are inside the deployment's network. Encryption is likewise taken from the connection rather than the configuration, since a tunnel can supply an already encrypted transport - PhysicalConnection.IsEncrypted covers both routes. Where there is no socket address at all - a tunnel, a custom transport, a Unix domain socket - this resolves to None rather than guessing: we ask for no address and reconnect the way we originally connected. 25 unit tests cover the matrix and the ranges: RFC1918 with its exact edges (172.15 and 172.32 excluded, 172.16-172.31 included), loopback, link-local, IPv6 ULA fc00::/7 with its edges, and IPv4-mapped addresses, which must be unwrapped or ::ffff:10.0.0.1 classifies as public. CGNAT 100.64/10 is deliberately not private, with the reasoning recorded. Verified live too: Auto over a public address with no TLS derived external-ip and the server answered 34.253.226.6. Note IsIPv6UniqueLocal is .NET 6+, so fc00::/7 is tested by hand - the full traversal build caught that, a filtered run would not have. ServerDefault remains the default deliberately. Our handling of a named successor does not yet move us onto the named node, so defaulting to Auto would populate a field we then use badly. Flip it with the named-target handoff. --- .../MaintenanceEndpointTypeResolver.cs | 90 +++++++++++++++++++ .../MaintenanceEndpointType.cs | 14 ++- src/StackExchange.Redis/PhysicalConnection.cs | 13 +++ .../PublicAPI/PublicAPI.Unshipped.txt | 11 +-- .../ServerEndPoint.Maintenance.cs | 17 +++- src/StackExchange.Redis/ServerEndPoint.cs | 2 +- .../MovingEndpointTypeScenarioTests.cs | 14 ++- .../MaintenanceEndpointTypeResolverTests.cs | 60 +++++++++++++ .../MaintenanceOptInClientTests.cs | 3 + 9 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs b/src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs new file mode 100644 index 000000000..b3c8ba459 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs @@ -0,0 +1,90 @@ +using System; +using System.Net; +using System.Net.Sockets; + +namespace StackExchange.Redis.Maintenance; + +/// +/// Chooses which moving-endpoint-type to ask a server for. +/// +/// +/// Two questions, answered independently. *Scope* comes from the address we are actually connected to: a private +/// or otherwise reserved address means we are inside the deployment's network and want the internal forms. +/// *Form* comes from whether the connection is encrypted: TLS implies the FQDN variants, because a certificate +/// generally cannot be validated against a bare address - so a client handed an IP mid-handoff would be unable +/// to verify the endpoint it was told to move to. +/// +/// +/// | private/reserved | otherwise +/// TLS off | internal-ip | external-ip +/// TLS on | internal-fqdn | external-fqdn +/// +/// +/// +/// Classifying the *connected* address matters, rather than the configured endpoint: the latter is usually a +/// hostname, and what decides whether we are inside the network is where it resolved to. Where there is no +/// socket address at all - a tunnel, a custom transport, a Unix domain socket - the honest answer is +/// : we cannot classify, so we ask for no address and reconnect the +/// way we originally connected. +/// +/// +internal static class MaintenanceEndpointTypeResolver +{ + /// + /// Derives the endpoint type for a connection. + /// + internal static MaintenanceEndpointType Derive(IPAddress? connectedAddress, bool isEncrypted) => + connectedAddress is null + ? MaintenanceEndpointType.None + : IsPrivateOrReserved(connectedAddress) + ? (isEncrypted ? MaintenanceEndpointType.InternalFqdn : MaintenanceEndpointType.InternalIp) + : (isEncrypted ? MaintenanceEndpointType.ExternalFqdn : MaintenanceEndpointType.ExternalIp); + + /// + /// Whether an address is private, or otherwise not routable on the public internet. + /// + /// + /// Covers RFC1918 (10/8, 172.16/12, 192.168/16), loopback, IPv4 link-local (169.254/16), IPv6 unique-local + /// (fc00::/7), IPv6 loopback and link-local, and IPv4-mapped IPv6 - which has to be unwrapped first, or an + /// address like ::ffff:10.0.0.1 classifies as public. + /// + /// CGNAT (100.64/10) is deliberately *not* treated as private. It is a genuine judgement call: it is not + /// publicly routable, but a client behind it is not inside the deployment's network either, which is the + /// question being asked here. Worth confirming against how the server classifies it before changing. + /// + /// + internal static bool IsPrivateOrReserved(IPAddress address) + { + if (address is null) throw new ArgumentNullException(nameof(address)); + + // unwrap ::ffff:a.b.c.d, so the IPv4 rules below actually apply to it + if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); + + if (IPAddress.IsLoopback(address)) return true; + + if (address.AddressFamily == AddressFamily.InterNetwork) + { + var bytes = address.GetAddressBytes(); + return bytes[0] switch + { + 10 => true, // 10.0.0.0/8 + 172 => bytes[1] >= 16 && bytes[1] <= 31, // 172.16.0.0/12 + 192 => bytes[1] == 168, // 192.168.0.0/16 + 169 => bytes[1] == 254, // 169.254.0.0/16, link-local + _ => false, + }; + } + + if (address.AddressFamily == AddressFamily.InterNetworkV6) + { + // IsIPv6UniqueLocal only exists from .NET 6, and this library targets down to net461 - so fc00::/7 + // is tested by hand. IsIPv6LinkLocal and IsIPv6SiteLocal are available everywhere. + if (address.IsIPv6LinkLocal || address.IsIPv6SiteLocal) return true; + + var v6 = address.GetAddressBytes(); + if ((v6[0] & 0xFE) == 0xFC) return true; // fc00::/7, unique-local + } + + return false; + } +} diff --git a/src/StackExchange.Redis/MaintenanceEndpointType.cs b/src/StackExchange.Redis/MaintenanceEndpointType.cs index 0711bfd87..8f0639e1c 100644 --- a/src/StackExchange.Redis/MaintenanceEndpointType.cs +++ b/src/StackExchange.Redis/MaintenanceEndpointType.cs @@ -1,4 +1,4 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.CodeAnalysis; using RESPite; namespace StackExchange.Redis; @@ -33,6 +33,18 @@ public enum MaintenanceEndpointType /// ServerDefault = 0, + /// + /// Work out the right form per connection, and ask for it. The recommended setting. + /// + /// + /// Derived from two facts about the connection as established: whether the address we actually reached is + /// private (so we want the internal forms) and whether the connection is encrypted (so we want the FQDN + /// forms, since a certificate generally cannot be validated against a bare address). Where there is no + /// socket address to classify - a tunnel, or a Unix domain socket - this resolves to + /// rather than guessing. + /// + Auto, + /// A private address, for a client inside the deployment's network. InternalIp, diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index c16cfa40a..7aed7c964 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -84,6 +84,19 @@ internal void GetBytes(out long sent, out long received) private Socket? _socket; internal Socket? VolatileSocket => Volatile.Read(ref _socket); + /// + /// Whether this connection is encrypted, however that came about. + /// + /// + /// Two routes, and both count: our own , or a tunnel-supplied transport that + /// reports it is already encrypted. Used to choose a moving-endpoint-type, where the question is + /// not "did the caller ask for TLS" but "is this connection actually encrypted" - because that is what + /// decides whether a certificate has to be validated against whatever address we are given next. + /// + internal bool IsEncrypted => + Volatile.Read(ref _transport)?.IsEncrypted == true + || Volatile.Read(ref _ioStream) is SslStream { IsEncrypted: true }; + // used for dummy test connections public PhysicalConnection( ConnectionType connectionType = ConnectionType.Interactive, diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 36d4b6364..5b9a0d5cc 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -119,11 +119,12 @@ StackExchange.Redis.StringIndex [SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SlotMigrations.get -> System.Collections.Generic.IReadOnlyList! [SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Time.get -> System.TimeSpan? [SER010]StackExchange.Redis.MaintenanceEndpointType -[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalFqdn = 4 -> StackExchange.Redis.MaintenanceEndpointType -[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalIp = 3 -> StackExchange.Redis.MaintenanceEndpointType -[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalFqdn = 2 -> StackExchange.Redis.MaintenanceEndpointType -[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalIp = 1 -> StackExchange.Redis.MaintenanceEndpointType -[SER010]StackExchange.Redis.MaintenanceEndpointType.None = 5 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.Auto = 1 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalFqdn = 5 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalIp = 4 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalFqdn = 3 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalIp = 2 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.None = 6 -> StackExchange.Redis.MaintenanceEndpointType [SER010]StackExchange.Redis.MaintenanceEndpointType.ServerDefault = 0 -> StackExchange.Redis.MaintenanceEndpointType [SER010]StackExchange.Redis.MaintenanceNotificationMode [SER010]StackExchange.Redis.MaintenanceNotificationMode.Auto = 2 -> StackExchange.Redis.MaintenanceNotificationMode diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 4823088db..ba982fca0 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -57,7 +57,22 @@ private bool ShouldRequestMaintenanceNotifications(bool isInteractive, bool nego /// /// The wire value for the configured moving-endpoint-type, or null to send no preference. /// - private RedisValue MaintenanceMovingEndpointTypeLiteral => Multiplexer.RawConfig.MaintenanceMovingEndpointType switch + private RedisValue MaintenanceMovingEndpointTypeLiteral(PhysicalConnection connection) + { + var configured = Multiplexer.RawConfig.MaintenanceMovingEndpointType; + if (configured == MaintenanceEndpointType.Auto) + { + // classify the address we actually reached, not the endpoint we dialled - the latter is usually a + // name, and where it resolved to is what decides whether we are inside the deployment's network + configured = MaintenanceEndpointTypeResolver.Derive( + (connection.VolatileSocket?.RemoteEndPoint as IPEndPoint)?.Address, + connection.IsEncrypted); + } + + return ToLiteral(configured); + } + + private static RedisValue ToLiteral(MaintenanceEndpointType type) => type switch { MaintenanceEndpointType.InternalIp => RedisLiterals.internal_ip, MaintenanceEndpointType.InternalFqdn => RedisLiterals.internal_fqdn, diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index e61c40eab..8807d1a67 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1373,7 +1373,7 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) // A bare ON leaves the endpoint type to the server, and every MOVING observed that way // carried no address at all - so when a caller asks for a specific form, say so. - var endpointType = MaintenanceMovingEndpointTypeLiteral; + var endpointType = MaintenanceMovingEndpointTypeLiteral(connection); msg = endpointType.IsNull ? Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON) : Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, [RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON, RedisLiterals.moving_endpoint_type, endpointType]); diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs index 4f318be14..9221bc7ad 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Threading.Tasks; @@ -33,6 +33,9 @@ public class MovingEndpointTypeScenarioTests(ExistingDatabaseFixture fixture, IT : IClassFixture { [Theory] + // Auto over a real socket to a public address with no TLS should derive external-ip, so the server should + // name an address rather than a hostname - which is the end-to-end proof of the derivation. + [InlineData(MaintenanceEndpointType.Auto)] [InlineData(MaintenanceEndpointType.ServerDefault)] [InlineData(MaintenanceEndpointType.ExternalFqdn)] [InlineData(MaintenanceEndpointType.ExternalIp)] @@ -107,6 +110,15 @@ public async Task DoesTheServerNameAReplacement(MaintenanceEndpointType type) log.WriteLine($"=> {type}: NewEndPoint = {push.NewEndPoint?.ToString() ?? "(null)"}; payload = {push.Payload ?? "(null)"}"); } + if (type == MaintenanceEndpointType.Auto) + { + // The scenario databases are reached over a public address with no TLS, so Auto should have + // derived external-ip: an address, not a hostname, and not null. + var named = moving[0].NewEndPoint; + Assert.NotNull(named); + Assert.IsType(named); + } + // The one thing worth asserting either way: whatever the server sent, we understood the frame. Assert.All(moving, push => Assert.Equal(MaintenanceNotificationType.Moving, push.NotificationType)); } diff --git a/tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs new file mode 100644 index 000000000..d240626cb --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs @@ -0,0 +1,60 @@ +using System.Net; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Deriving which moving-endpoint-type to ask for. +/// +/// +/// Worth testing exhaustively because it decides what a server sends us during a handoff, and getting the +/// scope wrong means being handed an address we cannot reach while getting the form wrong means being handed one +/// we cannot validate. +/// +public class MaintenanceEndpointTypeResolverTests +{ + [Theory] + // private, so the internal forms; TLS decides ip versus fqdn + [InlineData("10.0.0.1", false, MaintenanceEndpointType.InternalIp)] + [InlineData("10.0.0.1", true, MaintenanceEndpointType.InternalFqdn)] + // public, so the external forms + [InlineData("34.253.226.6", false, MaintenanceEndpointType.ExternalIp)] + [InlineData("34.253.226.6", true, MaintenanceEndpointType.ExternalFqdn)] + public void ScopeComesFromTheAddressAndFormFromTls(string address, bool encrypted, MaintenanceEndpointType expected) + => Assert.Equal(expected, MaintenanceEndpointTypeResolver.Derive(IPAddress.Parse(address), encrypted)); + + [Fact] + public void NoAddressMeansAskForNothing() + { + // A tunnel, a custom transport or a Unix domain socket gives us nothing to classify. "none" is the + // honest answer: we ask for no address and reconnect the way we originally connected, rather than + // guessing at a scope we cannot determine. + Assert.Equal(MaintenanceEndpointType.None, MaintenanceEndpointTypeResolver.Derive(null, isEncrypted: false)); + Assert.Equal(MaintenanceEndpointType.None, MaintenanceEndpointTypeResolver.Derive(null, isEncrypted: true)); + } + + [Theory] + [InlineData("10.0.0.1", true)] // RFC1918 10/8 + [InlineData("10.255.255.255", true)] + [InlineData("172.16.0.1", true)] // RFC1918 172.16/12 - lower edge + [InlineData("172.31.255.254", true)] // upper edge + [InlineData("172.15.0.1", false)] // just outside + [InlineData("172.32.0.1", false)] // just outside + [InlineData("192.168.1.1", true)] // RFC1918 192.168/16 + [InlineData("192.169.1.1", false)] // adjacent, and public + [InlineData("169.254.1.1", true)] // link-local + [InlineData("127.0.0.1", true)] // loopback + [InlineData("::1", true)] // IPv6 loopback + [InlineData("fe80::1", true)] // IPv6 link-local + [InlineData("fc00::1", true)] // IPv6 unique-local, lower edge of fc00::/7 + [InlineData("fdff::1", true)] // upper edge + [InlineData("fe00::1", false)] // outside fc00::/7 + [InlineData("::ffff:10.0.0.1", true)] // IPv4-mapped private: must be unwrapped, or it reads as public + [InlineData("::ffff:34.253.226.6", false)] // IPv4-mapped public + [InlineData("2001:4860:4860::8888", false)] // public IPv6 + [InlineData("34.253.226.6", false)] // public IPv4 + [InlineData("100.64.0.1", false)] // CGNAT: deliberately *not* private - see the remarks + public void ReservedRangesAreClassified(string address, bool expected) + => Assert.Equal(expected, MaintenanceEndpointTypeResolver.IsPrivateOrReserved(IPAddress.Parse(address))); +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs index 7a781de59..a47311c31 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -269,6 +269,9 @@ public async Task TheLogSaysWhetherTheFeatureIsLive(MaintenanceNotificationSuppo [InlineData(MaintenanceEndpointType.ExternalIp, "external-ip")] [InlineData(MaintenanceEndpointType.ExternalFqdn, "external-fqdn")] [InlineData(MaintenanceEndpointType.None, "none")] + // Auto over the in-process transport: there is no socket to classify, so it resolves to "none" rather than + // guessing at a scope. Over a real socket to a loopback address it would derive internal-ip. + [InlineData(MaintenanceEndpointType.Auto, "none")] public async Task MovingEndpointTypeIsSentWhenAskedFor(MaintenanceEndpointType type, string? expected) { // The point of asking: every MOVING observed on a real deployment carried no address, and every one of From fb7160a219f79ce5fc39a27f206bbf766ae5b064 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 16:22:30 +0100 Subject: [PATCH 06/36] Use a named successor directly, instead of hoping DNS agrees The previous handling of a named successor re-read the topology and recycled the dialled endpoint, and measurement showed that did not work: we recycled at +6.2s, landed back on the node being retired because DNS had not moved yet, and were closed at +21.6s anyway - the exact outcome the handoff exists to avoid. Now the endpoint's next connection attempt is pointed at the named address, and the connections are replaced. No DNS, which is the whole value of the field. Same scenario, same cluster, before and after: ServerDefault (DNS poll) MOVING +16.3s, handoff +16.3s, closed by server +35.3s ExternalFqdn (named) MOVING +5.9s, handoff +5.9s, never closed The important design choice is that this is a *connect target*, not a new endpoint in the collection. The ServerEndPoint keeps its identity, its place in server selection, and its TLS host - validation and SNI derive from ServerEndPoint.EndPoint, so moving the socket without moving the endpoint cannot perturb them. Adding the moved-to address as an endpoint would, and that is a documented trap in the cross-client contract. The target expires with the announced window and is cleared once a connection is established. Without the expiry, a server naming an address that turns out to be unreachable would pin the endpoint to it for the lifetime of the multiplexer, because every reconnect would retry the same dead target. The live test asserts the payoff rather than the mechanism: with an endpoint type requested, no SocketClosed occurs at all. That is the guard that would have caught the old behaviour, which produced a handoff that simply went nowhere. Full local suite 6250 passed; the three live handoff scenarios green. --- .../Maintenance/MaintenanceHandoff.cs | 19 ++++-- src/StackExchange.Redis/PhysicalConnection.cs | 10 +++ .../ServerEndPoint.Maintenance.cs | 65 +++++++++++++++++-- src/StackExchange.Redis/ServerEndPoint.cs | 3 + .../MovingHandoffScenarioTests.cs | 22 +++++-- .../MaintenanceHandoffTests.cs | 11 ++-- .../MaintenanceNotificationTests.cs | 15 ++++- 7 files changed, 122 insertions(+), 23 deletions(-) diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs index 38df5c310..f195d82ae 100644 --- a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs +++ b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -16,8 +16,14 @@ internal enum HandoffAction /// Drop our connections so they re-establish against the replacement address. Recycle, - /// The replacement is a different endpoint; the topology has to be re-read to find it. - Reconfigure, + /// + /// The server named where to go; point the next connection at it. + /// + /// + /// Not "add an endpoint and retire this one": the endpoint keeps its identity and its TLS host, and only + /// the socket target changes. See ServerEndPoint.HandoffTarget. + /// + MoveTo, } /// @@ -78,9 +84,10 @@ internal static async Task DecideAsync( { if (successor is not null) { - // Nothing is asserted about *which* endpoint: a successor may be an address we have never seen, so - // finding it is a topology question rather than a DNS one. - return new HandoffDecision(HandoffAction.Reconfigure, successor, "the server named a replacement endpoint"); + // Straight there: no DNS involved, which is the whole value of the field. Measured on RS 8.0.22, + // DNS trails a MOVING by 4.4s to 18.7s while the socket closes at 15.7s to 19.1s, so a named + // successor is the difference between moving immediately and possibly not moving in time at all. + return new HandoffDecision(HandoffAction.MoveTo, successor, "the server named a replacement endpoint"); } if (endpoint is not DnsEndPoint dns) diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 7aed7c964..d5272b28f 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -191,6 +191,16 @@ internal async Task BeginConnectAsync(ILogger? log) var rawConfig = bridge.Multiplexer.RawConfig; var tunnel = rawConfig.Tunnel; var connectTo = endpoint; + + // A MOVING may name where to go next; prefer it over resolving the endpoint again, since not + // waiting for DNS is the point. Note this changes only the *socket target*: the endpoint itself is + // untouched, so identity, server selection and - importantly - the TLS host and SNI (derived from + // ServerEndPoint.EndPoint below) all stay exactly as configured. + if (bridge.ServerEndPoint?.HandoffTarget is { } handoffTarget) + { + Trace($"handoff: connecting to {Format.ToString(handoffTarget)} in place of {Format.ToString(endpoint)}"); + connectTo = handoffTarget; + } if (tunnel is not null) { // A transport tunnel replaces the socket outright (the widest form of the existing diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index ba982fca0..a60811e6c 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -254,7 +254,62 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } + private volatile EndPoint? _handoffTarget; + private int _handoffTargetExpiryTicks; private int _handoffInFlight, _handoffRecycles; + + /// + /// Where the next connection attempt should go, when a server has named a replacement. + /// + /// + /// Deliberately a *connect* target rather than a new endpoint in the collection. This + /// keeps its identity, its place in server selection, and - the part that + /// matters most - its TLS host: certificate validation and SNI are derived from + /// , so moving the socket without moving the endpoint means a handoff + /// cannot perturb them. Adding the moved-to address as an endpoint would; that is a documented trap in the + /// cross-client contract. + /// + /// Expires, and that is not decoration. Without it, a server that named an address which turns out to be + /// unreachable would pin this endpoint to it for the lifetime of the multiplexer, because every reconnect + /// would keep trying the same dead target. On expiry we fall back to resolving the endpoint normally, which + /// is what we would have done anyway. + /// + /// + internal EndPoint? HandoffTarget + { + get + { + var target = _handoffTarget; + if (target is null) return null; + + if (unchecked(Environment.TickCount - Volatile.Read(ref _handoffTargetExpiryTicks)) >= 0) + { + _handoffTarget = null; // expired; resolve the endpoint the usual way from here on + return null; + } + + return target; + } + } + + /// + /// Points the next connection attempt at a named replacement, for as long as the announced window lasts. + /// + internal void SetHandoffTarget(EndPoint target, TimeSpan window) + { + Volatile.Write(ref _handoffTargetExpiryTicks, unchecked(Environment.TickCount + (int)Math.Max(window.TotalMilliseconds, 1000))); + _handoffTarget = target; + } + + /// + /// Forgets any handoff target, once a connection has been established. + /// + /// + /// Called on full establishment rather than on the connect attempt: if the attempt fails we want the next + /// one to try the target again, within its window. Once a connection is up, normal resolution resumes - + /// by then DNS has usually caught up anyway. + /// + internal void ClearHandoffTarget() => _handoffTarget = null; private volatile string? _lastHandoffOutcome; /// @@ -335,10 +390,12 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? case HandoffAction.Recycle: await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); break; - case HandoffAction.Reconfigure: - // A named successor is a different endpoint, so this is a topology question. Note no - // observed deployment has ever named one, so this path has never run outside a test. - Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, "moving names a successor"); + case HandoffAction.MoveTo when decision.Target is { } target: + // Point the next connection at the named address and replace the connections. Previously + // this only re-read the topology and recycled, which measurably did not work: we recycled + // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were + // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. + SetHandoffTarget(target, remaining); await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); break; default: diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 8807d1a67..2517f531b 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -944,6 +944,9 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // Clear the unselectable flag ASAP since we are open for business ClearUnselectable(UnselectableFlags.DidNotRespond); + // whatever a handoff pointed us at, we are connected now: resume normal resolution + ClearHandoffTarget(); + // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs index 556e33618..2587af7f2 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs @@ -116,12 +116,22 @@ void Note(string what) Assert.Equal(1, endpoint.HandoffRecycles); Assert.NotNull(endpoint.LastHandoffOutcome); - // Recycle *or* Reconfigure: which one depends on whether the server named a replacement, and it only - // does that when we asked for an endpoint type. Asserting "Recycle" alone was an assumption from the - // era when the field was always null. - Assert.True( - endpoint.LastHandoffOutcome.Contains("Recycle") || endpoint.LastHandoffOutcome.Contains("Reconfigure"), - $"unexpected handoff outcome: {endpoint.LastHandoffOutcome}"); + // Which outcome depends on whether the server named a replacement, and it only does that when we asked + // for an endpoint type: MoveTo when it did, Recycle when we had to find the new address via DNS. + var expectedOutcome = endpointType == MaintenanceEndpointType.ServerDefault ? "Recycle" : "MoveTo"; + Assert.Contains(expectedOutcome, endpoint.LastHandoffOutcome); + + if (endpointType != MaintenanceEndpointType.ServerDefault) + { + // The payoff, and the reason to ask for an endpoint type at all: with somewhere named to go we move + // immediately instead of waiting on DNS, so the server never has to close the connection out from + // under us. Measured before this worked: the handoff happened but landed back on the node being + // retired, and a SocketClosed followed ~15s later. + lock (failures) + { + Assert.DoesNotContain(ConnectionFailureType.SocketClosed, failures); + } + } Assert.True( await Poll.UntilAsync( diff --git a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs index 33ee7aeb6..079b5b588 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -68,10 +68,11 @@ public async Task AddressEndpointWithNoSuccessorHasNothingToDo() } [Fact] - public async Task NamedSuccessorAsksForAReconfigure() + public async Task NamedSuccessorIsUsedDirectly() { - // Never observed on any real deployment - eleven routes, all explicit nulls - so this exists because the - // contract has it, not because it fires. + // A named successor skips DNS entirely, which is the point of the field: DNS trails a MOVING by 4.4s to + // 18.7s while the socket closes at 15.7s to 19.1s, so waiting for it is sometimes waiting too long. + // Note the resolver here still reports the *old* address, and it is never consulted. var successor = new IPEndPoint(Replacement, 13486); var decision = await MaintenanceHandoff.DecideAsync( Hostname, successor, currentAddress: Retiring, @@ -79,7 +80,7 @@ public async Task NamedSuccessorAsksForAReconfigure() resolve: Resolves(Retiring), log: log.WriteLine); log.WriteLine(decision.ToString()); - Assert.Equal(HandoffAction.Reconfigure, decision.Action); + Assert.Equal(HandoffAction.MoveTo, decision.Action); Assert.Equal(successor, decision.Target); } diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 0fad1947f..f8e904b63 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -677,11 +677,22 @@ public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() lock (failures) failures.Add(e.FailureType); }; - server.SendMoving(null, timeSeconds: 2, newEndpoint: server.DefaultEndPoint, sequenceId: 0); + // a *different* address, so the handoff target is distinguishable from where we already are + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + server.SendMoving(null, timeSeconds: 2, newEndpoint: successor, sequenceId: 0); var moving = await events.NextAsync(); Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); - Assert.Equal(server.DefaultEndPoint, moving.NewEndPoint); + Assert.Equal(successor, moving.NewEndPoint); + + // The handoff should have recorded where to go next. Note the in-process transport routes by + // *endpoint* rather than by socket address, so the fake cannot observe the redirection itself - + // that is what the live scenario test covers. What is checked here is the intent. + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True( + await Poll.UntilAsync(() => endpoint.HandoffTarget is not null || endpoint.HandoffRecycles > 0, timeoutMilliseconds: 10_000), + "the handoff should have taken the named successor"); + log.WriteLine($"handoff target: {endpoint.HandoffTarget?.ToString() ?? "(cleared after reconnect)"}; recycles={endpoint.HandoffRecycles}"); // a fresh connection re-sends the opt-in, which is how a recycle is visible from the server's side Assert.True( From 7d8f80522547bd342a69b7de148966a12c3367d3 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 16:38:02 +0100 Subject: [PATCH 07/36] Default to deriving the moving-endpoint-type Auto becomes the default, so a handoff normally has somewhere named to go rather than having to wait for DNS - measured at 4 to 19 seconds behind the notification, against a socket that closes at 16 to 19 seconds. Safe to ask by default for a reason the contract states plainly: a server whose metadata lacks the parameter, or lacks the specific form requested, answers with a null endpoint rather than an error, which is exactly the behaviour of not asking. The request therefore costs nothing. All five forms were accepted by RS 8.0.22 in live runs. If a deployment is ever seen refusing the parameter outright, the fix is to remember that per server and fall back to a bare opt-in; nothing observed so far needs it, so that machinery is not being written on spec. One visible consequence, and the single test that changed: over a transport with no socket address - a tunnel, a Unix domain socket - Auto resolves to "none", so the opt-in now carries "moving-endpoint-type none" where it previously carried nothing. Both yield a null successor, so behaviour is unchanged; the wire is just more explicit about intent. Documented in docs/ServerMaintenanceEvent.md with the derivation table and the reason the TLS axis exists, since this is now on by default and worth being able to override or understand. --- docs/ServerMaintenanceEvent.md | 28 +++++++++++++++++++ .../Configuration/DefaultOptionsProvider.cs | 16 ++++++++--- .../MaintenanceOptInClientTests.cs | 8 ++++-- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 83de1563b..6995a7e5f 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -129,6 +129,34 @@ options.MaintenanceNotifications = MaintenanceNotificationMode.Auto; `Auto` is the right choice almost always: asking costs one command during the handshake, and a server that accepts and then never sends anything costs nothing at all. `Enabled` exists for the case where running without advance warning is worse than not running: it turns a silent absence into a startup failure, which also makes it a useful way to prove the feature is live in a staging environment. +### Asking where to go next + +When an endpoint is being replaced, the server can name its replacement - but only if asked. The client asks by +default (`maintMovingEndpointType=Auto`), working out the right form per connection: + +| | connected address is private | otherwise | +|---|---|---| +| **without TLS** | `internal-ip` | `external-ip` | +| **with TLS** | `internal-fqdn` | `external-fqdn` | + +The TLS split is about certificate validation: a certificate carrying DNS names cannot validate a bare address, +so an encrypted connection asks for names. Where there is no address to classify - a tunnel, a custom transport, +a Unix domain socket - the client asks for `none` rather than guessing, and falls back to reconnecting the way it +originally connected. + +This matters more than it sounds. Without a named replacement, a handoff has to wait for DNS to be repointed, +and DNS has been measured trailing the notification by anywhere from 4 to 19 seconds while the socket closes at +about 16 to 19 seconds - so on a bad run the connection is gone before DNS is ready. With a named replacement +the client moves within a second and the server never has to close anything. + +Override it if your deployment needs a specific form: + +``` +maintMovingEndpointType=ExternalFqdn +``` + +or `ServerDefault` to ask for nothing at all, which is what earlier versions did. + ## What the client does without your involvement | Notification | What the client does | diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index 8788892a1..e71795334 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -354,12 +354,20 @@ protected virtual string GetDefaultClientName() => /// Which form of address to ask a server to name when an endpoint moves. /// /// - /// - ask for nothing, and let the server decide. - /// A provider that knows how its deployment is reached can do better: an FQDN form is the right answer - /// wherever TLS is in play, because an address cannot be validated against a DNS-only certificate. + /// : derive it per connection, and ask. A bare opt-in leaves + /// the choice to the server, and measurement showed what that means in practice - every MOVING + /// observed that way named no replacement at all, so a handoff had to wait for DNS, which trails the + /// notification by anywhere from four to nineteen seconds. Asking produces an address immediately. + /// + /// Safe to ask for: a server whose metadata lacks the parameter, or lacks the specific form requested, + /// answers with a null endpoint rather than an error - which is exactly the behaviour of not asking. So + /// the downside of the request is nothing, and the upside is a handoff that does not race DNS. If a + /// deployment is ever seen *refusing* the parameter outright, the fix is to remember that per server and + /// fall back to a bare opt-in; nothing observed so far needs it. + /// /// [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] - public virtual MaintenanceEndpointType MaintenanceMovingEndpointType => MaintenanceEndpointType.ServerDefault; + public virtual MaintenanceEndpointType MaintenanceMovingEndpointType => MaintenanceEndpointType.Auto; /// /// Gets the value command timeouts are relaxed to during an announced disruption; 10 seconds, as the diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs index a47311c31..8c4fb6581 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -85,8 +85,12 @@ public async Task HandshakeOptsInWhenAuto() if (resp3) { - // a bare ON, so the server chooses; nothing here invents an endpoint type - Assert.Null(Assert.Single(OptedIn(server)).MovingEndpointType); + // The endpoint type now defaults to Auto, and over the in-process transport there is no socket + // address to classify - so it resolves to "none", meaning "send me no address, I will reconnect the + // way I connected". That is the honest answer where the scope cannot be determined, and it is what + // a tunnel or a Unix domain socket gets. Over a real socket this would be one of the four + // ip/fqdn forms; see MaintenanceEndpointTypeResolverTests. + Assert.Equal("none", Assert.Single(OptedIn(server)).MovingEndpointType); } // ...and the connection is entirely usable either way From c00791524d6fdb0da2039040ab7b0d1d63bc8557 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 1 Sep 2026 18:22:15 +0100 Subject: [PATCH 08/36] Cover mTLS, which also proves the TLS half of the derivation variant_index=2 provisions the mtls variant - the same TLS database plus enforce_client_authentication - and the setup response's mtls_files gives paths relative to the config directory: mtls/client.crt, mtls/client.key and mtls/ca_chain.pem. The part worth being careful about is that these are two different trust roots. The environment's ca.crt validates the *server*; the mtls/ material is the *client* identity, issued by the fault injector's own intermediate CA. Presented via SetUserPemCertificate, alongside TrustIssuer for the server side - and modelled as such, so a test cannot accidentally offer one where the other is wanted. Both variants pass against the live cluster, and mTLS also demonstrates the TLS axis of the endpoint-type derivation end to end: an encrypted connection derives an FQDN form, so MOVING named node2.: - a hostname the certificate can validate, rather than an address it could not. The test asserts the successor is a DnsEndPoint for that reason, which is the assertion that would catch the derivation silently reverting to an ip form. Also asserts the variant the setup actually built (single_tls versus mtls) and that the client material is present when required, so a mis-provisioned scenario fails as itself rather than as a handshake error. --- .../FaultInjector/ScenarioRun.cs | 41 +++++++++++++++++-- .../TlsScenarioTests.cs | 41 +++++++++++++++++-- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs index 6c90a2f6b..c3b7e1b96 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs @@ -56,8 +56,17 @@ private ScenarioRun(FaultInjectorClient injector, string scenario, string effect /// public ScenarioDatabase? Database { get; private set; } + /// The client-side material an mTLS database requires. + /// + /// Note these are the *client* identity, issued by the fault injector's own intermediate CA, and are a + /// different trust root from the certificate that validates the server: that one comes from the + /// environment's ca.crt. Conflating the two is the obvious way to get an mTLS test wrong. + /// The paths the setup response gives are relative to the config directory. + /// + public sealed record MtlsMaterial(string ClientCertificatePath, string ClientKeyPath, string CaChainPath); + /// The database a scenario provisioned for itself. - public sealed record ScenarioDatabase(string Name, int BdbId, string Host, int Port, bool Tls, string? Password, string? ProxyPolicy) + public sealed record ScenarioDatabase(string Name, int BdbId, string Host, int Port, bool Tls, string? Password, string? ProxyPolicy, MtlsMaterial? Mtls = null) { public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, policy {ProxyPolicy ?? "?"})"; @@ -82,6 +91,11 @@ public ConfigurationOptions GetClientConfig( options.SslHost = Host; options.TrustIssuer(environment.CertificateAuthorityPath ?? throw new InvalidOperationException($"{Name} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}")); + + // ...and if the database demands a client certificate, present one. Deliberately after + // TrustIssuer: that decides whether we accept the *server*, this decides what we offer about + // ourselves, and they use different trust roots. + if (Mtls is { } mtls) options.SetUserPemCertificate(mtls.ClientCertificatePath, mtls.ClientKeyPath); } return options; @@ -122,7 +136,7 @@ public static async Task SetupAsync( run.SetupResult = await injector.PostScenarioAsync(scenario, "setup", query, cancellationToken: cancellationToken); run.SetupId = FindString(run.SetupResult, "setup_id"); run.BdbId = FindInt(run.SetupResult, "bdb_id"); - run.Database = ReadDatabase(run.SetupResult, run.BdbId); + run.Database = ReadDatabase(run.SetupResult, run.BdbId, FaultInjectorEnvironment.Current?.ConfigDirectory.FullName); log($"setup complete: setup_id={run.SetupId ?? "(none)"} database={run.Database?.ToString() ?? "(none)"}"); return run; } @@ -178,7 +192,25 @@ public async ValueTask DisposeAsync() } } - private static ScenarioDatabase? ReadDatabase(JsonElement setup, int? bdbId) + /// + /// The mTLS material a setup reported, resolved against the config directory. + /// + private static MtlsMaterial? ReadMtls(JsonElement setup, string? configDirectory) + { + if (!setup.TryGetProperty("mtls_files", out var files) || files.ValueKind != JsonValueKind.Object) return null; + + var cert = FindString(files, "client_cert"); + var key = FindString(files, "client_key"); + var chain = FindString(files, "ca_chain"); + if (cert is null || key is null || chain is null || configDirectory is null) return null; + + return new MtlsMaterial( + System.IO.Path.Combine(configDirectory, cert), + System.IO.Path.Combine(configDirectory, key), + System.IO.Path.Combine(configDirectory, chain)); + } + + private static ScenarioDatabase? ReadDatabase(JsonElement setup, int? bdbId, string? configDirectory) { if (bdbId is not { } id) return null; @@ -198,7 +230,8 @@ public async ValueTask DisposeAsync() port, setup.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, FindString(setup, "password"), - FindString(setup, "config")); + FindString(setup, "config"), + ReadMtls(setup, configDirectory)); } private static string? FindString(JsonElement element, string name) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs index a99be3bc2..7d0e61153 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.IO; using System.Threading.Tasks; using StackExchange.Redis.Maintenance; @@ -29,8 +30,10 @@ namespace StackExchange.Redis.FaultInjector.Tests; public class TlsScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) : IClassFixture { - [Fact] - public async Task NotificationsArriveOverTlsAndIdentityIsVerified() + [Theory] + [InlineData(1, "single_tls", false)] + [InlineData(2, "mtls", true)] + public async Task NotificationsArriveOverTlsAndIdentityIsVerified(int variantIndex, string expectedConfig, bool expectClientCertificate) { fixture.RequireAvailable(); var cancellationToken = TestContext.Current.CancellationToken; @@ -56,10 +59,14 @@ public async Task NotificationsArriveOverTlsAndIdentityIsVerified() // trigger offers one variant ("single"), with include_tls two ("single", "single_tls"), and with // include_mtls a third ("mtls"). Passing include_tls alone provisions variant 0 and yields a // plaintext database, which is how this test first came to skip itself. + // include_tls / include_mtls widen the variant list; variant_index picks one. With no flags a + // trigger offers just "single"; include_tls adds "single_tls"; include_mtls adds "mtls", which is + // the same TLS database plus enforce_client_authentication. extra: new Dictionary { ["include_tls"] = "true", - ["variant_index"] = "1", + ["include_mtls"] = expectClientCertificate ? "true" : null, + ["variant_index"] = variantIndex.ToString(), }, cancellationToken: cancellationToken); @@ -72,6 +79,22 @@ public async Task NotificationsArriveOverTlsAndIdentityIsVerified() Assert.Skip("the injector provisioned a plaintext database despite include_tls=true; nothing to test here"); } + Assert.Equal(expectedConfig, database.ProxyPolicy); // the setup reports which variant it built + if (expectClientCertificate) + { + // A database with enforce_client_authentication rejects a connection that offers no certificate, so + // the material has to be there: this is the client identity, issued by the injector's own + // intermediate CA, and a different trust root from the one validating the server. + Assert.NotNull(database.Mtls); + log.WriteLine($"presenting client certificate {database.Mtls.ClientCertificatePath}"); + Assert.True(File.Exists(database.Mtls.ClientCertificatePath), $"missing {database.Mtls.ClientCertificatePath}"); + Assert.True(File.Exists(database.Mtls.ClientKeyPath), $"missing {database.Mtls.ClientKeyPath}"); + } + else + { + Assert.Null(database.Mtls); + } + var clock = Stopwatch.StartNew(); var events = new List(); @@ -137,6 +160,18 @@ public async Task NotificationsArriveOverTlsAndIdentityIsVerified() { log.WriteLine($" {events.Count} notification(s) over TLS"); Assert.NotEmpty(events); + + // The TLS half of the endpoint-type derivation, end to end: an encrypted connection asks for an + // FQDN form, so a MOVING should name a *host* rather than an address - which is the whole point, + // since a certificate carrying DNS names cannot validate a bare IP. + foreach (var moving in events.Where(e => e.NotificationType == MaintenanceNotificationType.Moving)) + { + log.WriteLine($" MOVING named: {moving.NewEndPoint?.ToString() ?? "(null)"}"); + if (moving.NewEndPoint is not null) + { + Assert.IsType(moving.NewEndPoint); + } + } } // and the TLS handshake succeeds again on the *replacement* connection, which is the part a From 8f6577f4dbec322b849a722d2dbb5a1986fc2531 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 10:17:46 +0100 Subject: [PATCH 09/36] Apply the half-window reconnect rule where it is the only tool The contract asks a client with no named replacement to "schedule a graceful reconnect to its currently configured endpoint after half of the grace period is over". Implemented, but deliberately not uniformly, because measurement says a blind clock-based reconnect is usually premature: at half of a 15s window, DNS had moved in one of three observed runs (+4.4s) and lagged well past it in the others (+9.7s, +18.7s). Reconnecting on the clock would normally land straight back on the node being retired. So the handoff now distinguishes the cases: successor named go there immediately hostname, address visible poll DNS, recycle when it moves hostname, DNS never moves in window do nothing; the close is handled normally address endpoint, no successor recycle at half the window no visible address (tunnel, UDS) recycle at half the window The last two are what the rule was written for. A change there is undetectable from the client, yet the address may be a stable front for a backend that has already moved - and waiting passively means being closed mid-command instead of choosing the moment. Both previously did nothing at all. Tests cover each branch, including the deliberate divergence, so the reasoning is pinned rather than just written down: polling beats the clock where an address is visible, and doing nothing when DNS never moves is a decision rather than an oversight. Worth raising upstream: the advice would be better as "at half the grace period or when the endpoint resolves elsewhere, whichever comes first" - on this deployment the server-side endpoint moves at +8.6s of a 15s window and DNS trails it, so the letter of the rule fires before either has happened. --- .../Maintenance/MaintenanceHandoff.cs | 27 ++++++++++++-- .../ServerEndPoint.Maintenance.cs | 7 ++++ .../MaintenanceHandoffTests.cs | 37 ++++++++++++++----- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs index f195d82ae..960b501d0 100644 --- a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs +++ b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs @@ -13,6 +13,24 @@ internal enum HandoffAction /// Nothing useful is available; let the server close the socket and reconnect then. None, + /// + /// Replace the connections once half the announced window has passed, without looking for a better target. + /// + /// + /// The contract's rule for a notification that names no replacement: "schedule a graceful reconnect to its + /// currently configured endpoint after *half* of the grace period is over" - not immediately, and not at the + /// deadline. Used only where there is nothing better to go on, because measurement shows it is premature + /// when there is: at half of a 15s window, DNS had moved in one of three observed runs, so reconnecting on + /// the clock alone would usually land back on the node being retired. + /// + /// Where it *is* right: an address endpoint, or a connection whose address we cannot see. Those cannot be + /// re-resolved, so a change is undetectable from here - but the address may well be a stable front for a + /// backend that has already moved, which is exactly the case the rule was written for. Waiting passively + /// instead means being closed mid-command rather than choosing the moment. + /// + /// + RecycleAtHalfWindow, + /// Drop our connections so they re-establish against the replacement address. Recycle, @@ -93,16 +111,17 @@ internal static async Task DecideAsync( if (endpoint is not DnsEndPoint dns) { return new HandoffDecision( - HandoffAction.None, + HandoffAction.RecycleAtHalfWindow, null, $"{Format.ToString(endpoint)} is an address, not a name, and no replacement was named: nothing to re-resolve"); } if (currentAddress is null) { - // Without knowing where we are, "has it moved" is unanswerable - and guessing would mean recycling - // onto whatever DNS says right now, which for the first several seconds is the address being retired. - return new HandoffDecision(HandoffAction.None, null, "the address of the current connection is unknown"); + // Without knowing where we are, "has it moved" is unanswerable, so there is nothing to poll for - + // which is precisely when the contract's half-window reconnect is the right tool. + return new HandoffDecision( + HandoffAction.RecycleAtHalfWindow, null, "the address of the current connection is unknown"); } var replacement = await AdvertisedAddressProbe.ProbeAsync( diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index a60811e6c..c23030cb1 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -390,6 +390,13 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? case HandoffAction.Recycle: await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); break; + case HandoffAction.RecycleAtHalfWindow: + // The contract's rule for "no replacement named", applied where there is nothing better to + // go on. Half of the *announced* window, less whatever the jitter already spent. + var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; + if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); + await DrainThenRecycleAsync(window - jitter - (half > TimeSpan.Zero ? half : TimeSpan.Zero), decision.Reason).ForAwait(); + break; case HandoffAction.MoveTo when decision.Target is { } target: // Point the next connection at the named address and replace the connections. Previously // this only re-read the topology and recycled, which measurably did not work: we recycled diff --git a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs index 079b5b588..2326e3288 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs @@ -53,18 +53,19 @@ public async Task HostnameThatNeverMovesDoesNothing() } [Fact] - public async Task AddressEndpointWithNoSuccessorHasNothingToDo() + public async Task AddressEndpointWithNoSuccessorReconnectsOnTheClock() { - // An address cannot be re-resolved, and nothing was named: there is no handoff to make. This is the - // case a cluster deployment would hit, and it is why MOVING must not simply reuse endpoint retirement - - // there would be nothing to retire *to*. + // An address cannot be re-resolved and nothing was named, so a change is undetectable from here - which + // is exactly the case the contract's half-window rule was written for. The address may be a stable + // front for a backend that has already moved, and waiting passively means being closed mid-command + // instead of choosing the moment. var decision = await MaintenanceHandoff.DecideAsync( new IPEndPoint(Retiring, 13486), successor: null, currentAddress: Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), resolve: Resolves(Replacement), log: log.WriteLine); log.WriteLine(decision.ToString()); - Assert.Equal(HandoffAction.None, decision.Action); + Assert.Equal(HandoffAction.RecycleAtHalfWindow, decision.Action); } [Fact] @@ -85,17 +86,35 @@ public async Task NamedSuccessorIsUsedDirectly() } [Fact] - public async Task UnknownCurrentAddressDoesNothingRatherThanGuessing() + public async Task UnknownCurrentAddressReconnectsOnTheClockRatherThanPolling() { - // Recycling here would mean accepting whatever DNS says *now*, which for the first several seconds is - // the address being retired - so we would hand off to the node we were told to leave. + // With no idea where we are, "has it moved?" is unanswerable, so there is nothing to poll for. A + // tunnel or a Unix domain socket lands here, and for a tunnel the target genuinely may have moved + // underneath us - so the half-window reconnect is the only tool, and better than waiting to be closed. var decision = await MaintenanceHandoff.DecideAsync( Hostname, successor: null, currentAddress: null, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), resolve: Resolves(Replacement), log: log.WriteLine); log.WriteLine(decision.ToString()); - Assert.Equal(HandoffAction.None, decision.Action); + Assert.Equal(HandoffAction.RecycleAtHalfWindow, decision.Action); + } + + [Fact] + public async Task PollingBeatsTheClockWhenWeCanSeeTheAddress() + { + // The deliberate divergence from the contract's half-window rule, and the reason for it. Where DNS can + // be polled we wait for it to actually move rather than reconnecting on a timer: measured across three + // runs, DNS had moved by half of a 15s window only once (+4.4s), and lagged well past it otherwise + // (+9.7s, +18.7s) - so reconnecting on the clock would usually land back on the node being retired. + // Doing nothing when it never moves is also deliberate: the server closes the socket, the reconnect + // re-resolves, and the relaxed window covers the gap. + var stillOld = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromMilliseconds(120), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: Resolves(Retiring), log: log.WriteLine); + + Assert.Equal(HandoffAction.None, stillOld.Action); } [Theory] From 22ebea434283fe78eaa76167a5a310318b8dc138 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 10:37:29 +0100 Subject: [PATCH 10/36] Serialise the fault-injector tier, and stop bounding it with an HTTP timeout The first time the whole suite ran together it failed 20 of 26 - every one of which passes individually. Two harness defects, neither visible while running one class at a time. The HttpClient had a two-minute timeout, and the injector *queues* actions: with several classes in flight, a scenario setup that takes twelve seconds alone sits for minutes, so the request was cancelled and reported as "TaskCanceledException: the configured HttpClient.Timeout of 120 seconds elapsing" - which says nothing about what actually happened. Removed: bounding belongs to the caller's cancellation token and to WaitForActionAsync, which can distinguish "still working" from "wedged". A blanket client timeout cannot. And the tier now runs strictly serially. There is one cluster and one injector, and the scenarios mutate *cluster* state - node exclusions, maintenance mode, endpoint policies - so parallel classes interfere semantically as well as starving each other; the run produced "Need at least 2 nodes with shards for slot-shuffle" for exactly that reason. Serial costs wall-clock, since the scenarios are minutes each, and buys results that mean something. Also broadens the port-collision retry, which was the one real failure in the aborted run before it: Redis Enterprise answers "port_unavailable" with the prose "Unavailable or invalid port", matching none of the phrases the retry looked for, so a retryable collision failed a whole fixture instead of moving up a port. The base port moves to 14500 as well, clear of the 13xxx range the scenario setups pick from, so our own databases are not competing for ports in the first place. --- .../AssemblyInfo.cs | 11 +++++++++++ .../Environment/FaultInjectorFixture.cs | 19 +++++++++++++------ .../FaultInjector/FaultInjectorClient.cs | 10 +++++++++- 3 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs new file mode 100644 index 000000000..40188bd5b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs @@ -0,0 +1,11 @@ +using Xunit; + +// One cluster, one injector, and scenarios that mutate *cluster* state - node exclusions, maintenance mode, +// endpoint policies. Running classes in parallel therefore breaks two ways at once: they interfere semantically +// (one scenario's teardown restores nodes another is relying on, giving errors like "Need at least 2 nodes with +// shards"), and they starve each other, because the injector processes actions through a queue. Measured: the +// first whole-suite run failed 20 of 26, almost all of them waiting on a queued setup. +// +// So this tier is strictly serial. It costs wall-clock - the scenarios are minutes each - and buys results that +// mean something. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs index 6406b8b88..94e996c35 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -107,7 +107,9 @@ public async ValueTask DisposeAsync() /// private async Task CreateDatabaseAsync() { - const int BasePort = 13500, Attempts = 8; + // Well clear of the 13xxx range the scenario setups pick from, so our own databases are not competing + // with theirs for ports in the first place. + const int BasePort = 14500, Attempts = 8; var name = $"{NamePrefix}{Shape.Label}-{RunId}"; Exception? last = null; @@ -143,11 +145,16 @@ private async Task CreateDatabaseAsync() /// "missing shard_key_regex" under eight identical tracebacks. /// private static bool IsPortCollision(Exception ex) => - ex.Message.Contains("port", StringComparison.OrdinalIgnoreCase) - && (ex.Message.Contains("in use", StringComparison.OrdinalIgnoreCase) - || ex.Message.Contains("already", StringComparison.OrdinalIgnoreCase) - || ex.Message.Contains("taken", StringComparison.OrdinalIgnoreCase) - || ex.Message.Contains("conflict", StringComparison.OrdinalIgnoreCase)); + // "port_unavailable" is what Redis Enterprise actually answers, with the prose "Unavailable or invalid + // port" - which matched none of the phrases the first version of this looked for, so a perfectly + // retryable collision failed the whole fixture instead of moving up a port. + ex.Message.Contains("port_unavailable", StringComparison.OrdinalIgnoreCase) + || (ex.Message.Contains("port", StringComparison.OrdinalIgnoreCase) + && (ex.Message.Contains("in use", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("already", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("taken", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("unavailable", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("conflict", StringComparison.OrdinalIgnoreCase))); /// /// Best-effort removal of databases left behind by earlier runs. diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs index 3bbc96e30..d467ba96c 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs @@ -19,7 +19,15 @@ namespace StackExchange.Redis.FaultInjector.Tests; /// public sealed class FaultInjectorClient(Uri baseAddress) : IDisposable { - private readonly HttpClient _http = new() { BaseAddress = baseAddress, Timeout = TimeSpan.FromMinutes(2) }; + /// + /// No , deliberately. The injector *queues* actions, so a request can + /// legitimately sit for many minutes when anything else is in flight - and a two-minute limit here failed + /// twenty of twenty-six tests the first time the whole suite ran together, all of them reported as a + /// cancelled HTTP request rather than as what they were. Bounding belongs to the caller's cancellation + /// token and to , which can tell "still working" from "wedged"; a blanket + /// client timeout cannot. + /// + private readonly HttpClient _http = new() { BaseAddress = baseAddress, Timeout = Timeout.InfiniteTimeSpan }; /// /// Statuses that mean "still going". Both of them, which is the point. From b5e60618ba988004d0ff8f65c0c0dc0fe0fa2467 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 11:20:10 +0100 Subject: [PATCH 11/36] Sweep the databases a killed scenario leaves behind Cancelling an HTTP request does not cancel the server's work. The setup calls that timed out client-side had already created their databases, and because SetupAsync threw there was no ScenarioRun to dispose - so no teardown ran, and the setup_id was never learned, leaving nothing to clean up with. 22 orphaned databases, holding ports and shards, cleared by hand. The startup sweep only knew about databases we create ourselves (sertest-), not the ones a scenario's setup leg creates and names for itself (tcs-, sm-), so nothing self-healed. It now sweeps those prefixes too, which makes an interrupted run recoverable instead of a manual job. The template databases are protected by *bdb id* rather than by name: endpoints.json keys them by their configured name, and relying on that key equalling the database's actual name is not an assumption worth making when the consequence of being wrong is deleting the wrong database. Verified: the failover fixture provisions on the new base port, passes, and the cluster is left with exactly its two original databases. Full tier 26/26 green serialised, 36 minutes. --- .../Environment/FaultInjectorFixture.cs | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs index 94e996c35..8ea143f8e 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -33,6 +34,23 @@ public abstract class FaultInjectorFixture(DatabaseShape shape) : IAsyncLifetime /// public const string NamePrefix = "sertest-"; + /// + /// Name prefixes the sweep will remove, beyond our own. + /// + /// + /// A scenario's setup leg creates its *own* database, named by the injector - tcs- for + /// topology-change, sm- for slot-migrate - so those leaks are not ours to name but are ours to clean + /// up. And they do leak: cancelling the setup request does not cancel the injector's work, so a run killed + /// mid-setup leaves a database nothing holds a handle to. That happened, 22 times, and needed clearing by + /// hand because the sweep only knew about databases we had created ourselves. + /// + /// Safe on the assumption this is a dedicated test environment - which the tier already assumes, since it + /// creates and destroys databases and reshapes the cluster. Databases named in endpoints.json (the + /// template's own) are never touched, whatever their prefix. + /// + /// + private static readonly string[] SweepablePrefixes = [NamePrefix, "tcs-", "sm-"]; + private static readonly string RunId = Guid.NewGuid().ToString("n")[..6]; private FaultInjectorClient? _injector; @@ -176,10 +194,16 @@ private async Task SweepOrphansAsync() try { + // Never touch the environment's own databases. Matched by bdb id rather than by name: endpoints.json + // keys them by the name they were configured with, and relying on that key equalling the database's + // actual name is an assumption worth not making when the consequence is deleting the wrong thing. + var template = ExistingDatabase.ReadAll(Environment).Values.Select(d => d.BdbId).ToHashSet(); + using var rest = new ClusterRestClient(cluster, Environment.CertificateAuthorityPath); foreach (var (bdbId, name) in await rest.ListDatabasesAsync()) { - if (!name.StartsWith(NamePrefix, StringComparison.Ordinal)) continue; // never anything but ours + if (template.Contains(bdbId)) continue; + if (!SweepablePrefixes.Any(prefix => name.StartsWith(prefix, StringComparison.Ordinal))) continue; Console.WriteLine($"sweeping orphaned test database {name} (bdb {bdbId})"); await Injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = bdbId }); From e9830ac21b891243444ae79869bf2332733d37af Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 11:43:01 +0100 Subject: [PATCH 12/36] D9: a soak for the state that only misbehaves with repetition toys/MaintenanceSoak drives continuous traffic while notifications are injected on a loop, and asserts the invariants that need repetition to fail: a relaxed window that never closes, a handoff flag or target that is never cleared, the event collapse silently eating or duplicating events, connections or memory accumulating. None of those show up once; all of them show up on the thousandth cycle, which is why unit tests cannot reach them. Hosted over a real TCP socket rather than the tunnel the test suite uses. A soak is looking for what leaks over thousands of cycles - sockets, pipes, buffers - and the tunnel bypasses the machinery most likely to leak. It also gives the connection a genuine remote address, so the endpoint-type derivation runs for real instead of resolving to "nothing to classify". 5000 cycles: 25.2M commands with 64 failures (all self-inflicted, since the soak severs connections on purpose), 200 handoffs, memory flat at ~700KB across the run, one live client throughout and 201 over the run - so nothing accumulates. Two of the invariants were wrong before they were right, which is worth recording: - "the relaxed window closed" failed on every checkpoint of a healthy run, because notifications were arriving every 25ms and a window that keeps being extended is correct behaviour, not a leak. It now asserts the window is open mid-storm and closes once injection pauses - two useful checks instead of one meaningless one. - "every notification was raised" reported a loss of one in 320, because it counted calls rather than deliveries; a MOVING handoff replaces the connection, so a notification issued at that instant has nobody to send to. Now counts what the server delivered, and tolerates loss up to the number of connection replacements - a frame queued for a socket that is being replaced is genuinely lost, and anything beyond that is a real leak. Over-raising is never tolerated, since that would mean the collapse has stopped collapsing. --- .../StackExchange.Redis.csproj | 2 + toys/MaintenanceSoak/Invariants.cs | 50 ++++ toys/MaintenanceSoak/MaintenanceSoak.csproj | 21 ++ toys/MaintenanceSoak/Program.cs | 244 ++++++++++++++++++ toys/MaintenanceSoak/SoakServer.cs | 110 ++++++++ 5 files changed, 427 insertions(+) create mode 100644 toys/MaintenanceSoak/Invariants.cs create mode 100644 toys/MaintenanceSoak/MaintenanceSoak.csproj create mode 100644 toys/MaintenanceSoak/Program.cs create mode 100644 toys/MaintenanceSoak/SoakServer.cs diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index e799f510a..f808312da 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -61,6 +61,8 @@ + + diff --git a/toys/MaintenanceSoak/Invariants.cs b/toys/MaintenanceSoak/Invariants.cs new file mode 100644 index 000000000..6888aa6f4 --- /dev/null +++ b/toys/MaintenanceSoak/Invariants.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace StackExchange.Redis.MaintenanceSoak; + +/// +/// The things that must stay true however many cycles run. +/// +/// +/// These are the failure modes a unit test structurally cannot reach, because each needs *repetition* to +/// appear: state that accumulates, a flag that is set and never cleared, a window that is extended once too +/// often. Each violation is recorded with the cycle it happened on rather than throwing, so one run reports +/// everything it found instead of the first thing. +/// +internal sealed class Invariants +{ + private readonly List _violations = []; + + public IReadOnlyList Violations => _violations; + + public void Check(bool condition, int cycle, string what) + { + if (!condition) _violations.Add($"cycle {cycle}: {what}"); + } + + /// + /// Whether the same violation has already been recorded, so a systemic failure reports once per kind + /// rather than once per cycle. + /// + public bool AlreadySeen(string what) => _violations.Exists(v => v.EndsWith(what, StringComparison.Ordinal)); + + public void Record(int cycle, string what) + { + if (!AlreadySeen(what)) _violations.Add($"cycle {cycle}: {what}"); + } +} + +/// +/// A memory sample, taken with a forced collection so the number means something. +/// +internal readonly record struct MemorySample(int Cycle, long Bytes) +{ + public static MemorySample Take(int cycle) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + return new MemorySample(cycle, GC.GetTotalMemory(forceFullCollection: true)); + } +} diff --git a/toys/MaintenanceSoak/MaintenanceSoak.csproj b/toys/MaintenanceSoak/MaintenanceSoak.csproj new file mode 100644 index 000000000..0591dff3c --- /dev/null +++ b/toys/MaintenanceSoak/MaintenanceSoak.csproj @@ -0,0 +1,21 @@ + + + + Exe + net10.0 + enable + StackExchange.Redis.MaintenanceSoak + + true + + + $(NoWarn);SER010;StringToRedisValue + + + + + + + + diff --git a/toys/MaintenanceSoak/Program.cs b/toys/MaintenanceSoak/Program.cs new file mode 100644 index 000000000..a7d16999c --- /dev/null +++ b/toys/MaintenanceSoak/Program.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; +using StackExchange.Redis.MaintenanceSoak; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Server; +using static StackExchange.Redis.Server.RedisServer; + +// Soak for the maintenance-notification machinery: continuous traffic while notifications are injected on a +// loop, watching for the things that only appear with repetition. +// +// Why this exists, when the feature already has unit tests and a live scenario tier: stage 2 onwards introduced +// real state with lifetimes - relaxed windows that extend, a post-event tail, an eight-slot dedup ring, a +// handoff in-flight flag, a handoff target with an expiry - and its failure modes are "a window never closes", +// "a flag is never cleared", "something accumulates". None of those show up once; all of them show up on the +// thousandth cycle. That is the gap this fills, and it is what discharges NF.1. +// +// Usage: MaintenanceSoak [cycles] [--workers N] [--port N] + +int cycles = 500, workers = 8, port = 0; +for (int i = 0; i < args.Length; i++) +{ + if (args[i] == "--workers" && i + 1 < args.Length) workers = int.Parse(args[++i]); + else if (args[i] == "--port" && i + 1 < args.Length) port = int.Parse(args[++i]); + else if (int.TryParse(args[i], out var parsed)) cycles = parsed; +} + +Console.WriteLine($"soak: {cycles} cycles, {workers} workers"); + +var invariants = new Invariants(); +await using var host = new SoakServer(new MemoryCacheRedisServer(), port); +Console.WriteLine($"server listening on {host.EndPoint}"); + +var options = new ConfigurationOptions +{ + EndPoints = { host.EndPoint }, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = MaintenanceNotificationMode.Enabled, + AbortOnConnectFail = false, + ConnectTimeout = 5_000, + SyncTimeout = 5_000, + AllowAdmin = true, + + // Short windows on purpose. The defaults are a 10s floor with a 20s tail, which is right for a real + // deployment and useless here: the soak injects continuously, so it would spend its whole life inside one + // window and never observe one *closing*. One second exercises the same code with a tractable clock. + MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1), +}; + +await using var muxer = await ConnectionMultiplexer.ConnectAsync(options); +var server = ((IInternalConnectionMultiplexer)muxer).GetServerEndPoint(host.EndPoint); +Console.WriteLine($"connected; maintenance notifications active: {server.MaintenanceNotificationsActive}"); +if (!server.MaintenanceNotificationsActive) +{ + Console.Error.WriteLine("FAIL: the opt-in was not accepted, so this run would prove nothing"); + return 2; +} + +// ---- continuous traffic ------------------------------------------------------------------------------------- +using var running = new CancellationTokenSource(); +long commands = 0, failures = 0; +var traffic = Enumerable.Range(0, workers).Select(worker => Task.Run(async () => +{ + var db = muxer.GetDatabase(); + var key = (RedisKey)$"soak-{worker}"; + while (!running.IsCancellationRequested) + { + try + { + await db.StringSetAsync(key, Guid.NewGuid().ToString("n")); + await db.StringGetAsync(key); + Interlocked.Add(ref commands, 2); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // Expected: the soak severs connections on purpose. Counted, not fatal - what matters is that + // traffic recovers, which the throughput check below covers. + Interlocked.Increment(ref failures); + await Task.Delay(10); + } + } +})).ToArray(); + +// ---- notification counting --------------------------------------------------------------------------------- +long events = 0; +muxer.ServerMaintenanceEvent += (_, e) => +{ + if (e is PushMaintenanceEvent) Interlocked.Increment(ref events); +}; + +// ---- the loop ---------------------------------------------------------------------------------------------- +var baseline = MemorySample.Take(0); +var samples = new List { baseline }; +var watch = Stopwatch.StartNew(); +long expectedEvents = 0; +int seq = 0; + +for (int cycle = 1; cycle <= cycles; cycle++) +{ + // Rotate through the shapes, so no single kind dominates and the pairs interleave. + // + // Note every Send returns how many opted-in clients it actually reached, and that is what counts as "sent" + // - not the number of calls. A MOVING handoff deliberately replaces the connection, so a notification + // issued during that instant has nobody to go to and returns zero. Counting calls instead reported a lost + // event on a run that had lost nothing. + switch (cycle % 5) + { + case 0: + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 1, shardIds: "[\"1\"]", sequenceId: seq++); + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: null, shardIds: "[\"1\"]", sequenceId: seq++); + break; + case 1: + expectedEvents += host.Server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, "0-100", sequenceId: seq++); + expectedEvents += host.Server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [($"{host.EndPoint.Address}:{host.EndPoint.Port}", $"{host.EndPoint.Address}:{host.EndPoint.Port}", "0-100")], sequenceId: seq++); + break; + case 2: + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 1, shardIds: "[\"2\"]", sequenceId: seq++); + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, timeSeconds: null, shardIds: "[\"2\"]", sequenceId: seq++); + break; + case 3: + // MOVING with no successor: the DNS/half-window path, and it replaces connections + expectedEvents += host.Server.SendMoving(null, timeSeconds: 1, newEndpoint: null, sequenceId: seq++); + break; + default: + // MOVING naming somewhere to go: the handoff-target path + expectedEvents += host.Server.SendMoving(null, timeSeconds: 1, newEndpoint: host.EndPoint, sequenceId: seq++); + break; + } + + await Task.Delay(25); + + if (cycle % 50 == 0) + { + // Mid-storm the window *should* be open: notifications are arriving faster than it can expire, so an + // endpoint that is not relaxed here means they are being received and ignored. + invariants.Check(server.IsMaintenanceRelaxed, cycle, "timeouts were not relaxed during continuous notifications"); + + // ...and then it must close once they stop. This is the check that needs the quiet: asking whether a + // window has closed while still injecting only measures the injection rate, which is how the first + // version of this reported a violation on every checkpoint of a perfectly healthy run. + var settled = await SettlesAsync(() => !server.IsMaintenanceRelaxed, TimeSpan.FromSeconds(10)); + invariants.Check(settled, cycle, "the relaxed window never closed after notifications stopped (a stuck timeout)"); + + // A handoff must never be left in flight either: the flag is what stops a second one starting, so a + // leak means no future MOVING is ever acted on again - silently. + invariants.Check(server.HandoffTarget is null, cycle, "a handoff target outlived its window (would pin this endpoint to one address)"); + + // the dedup ring is fixed-size, so what is worth checking is that collapsing still *works* after + // thousands of events rather than silently dropping everything + var seen = Interlocked.Read(ref events); + invariants.Check(seen > 0, cycle, "no notifications were raised at all"); + + samples.Add(MemorySample.Take(cycle)); + var clients = host.Server.ClientCount; + Console.WriteLine( + $" cycle {cycle,5}: events={seen,6} commands={Interlocked.Read(ref commands),8} " + + $"failures={Interlocked.Read(ref failures),4} clients={clients,2} handoffs={server.HandoffRecycles,4} " + + $"mem={samples[^1].Bytes / 1024,7}KB"); + + // A recycle closes the old connection and opens one; the fake should not be accumulating them. + invariants.Check(clients <= 8, cycle, $"the server is holding {clients} clients, which suggests connections are not being released"); + } +} + +running.Cancel(); +await Task.WhenAll(traffic); +watch.Stop(); + +// ---- report ------------------------------------------------------------------------------------------------- +var first = samples.Count > 1 ? samples[1] : baseline; // after warmup +var last = samples[^1]; +var growth = first.Bytes == 0 ? 0 : (last.Bytes - first.Bytes) * 100.0 / first.Bytes; + +Console.WriteLine(); +Console.WriteLine($"cycles: {cycles} in {watch.Elapsed.TotalSeconds:0.0}s"); +Console.WriteLine($"notifications: {Interlocked.Read(ref events)} raised, {expectedEvents} delivered"); +Console.WriteLine($"commands: {Interlocked.Read(ref commands)} ok, {Interlocked.Read(ref failures)} failed"); +Console.WriteLine($"memory: {first.Bytes / 1024}KB -> {last.Bytes / 1024}KB ({growth:+0.0;-0.0;0}%)"); +Console.WriteLine($"clients: {host.Server.ClientCount} now, {host.Server.TotalClientCount} over the run"); + +// Far fewer than the number of MOVINGs sent, and that is correct: a MOVING arriving while a handoff is already +// in flight is ignored, since starting a second poll against the same endpoint achieves nothing. Reported so +// the number is visible rather than assumed - a zero here would mean the handoff path never ran at all. +Console.WriteLine($"handoffs: {server.HandoffRecycles} connection replacements"); + +// Growth is reported always and failed only when egregious: a soak that cries wolf on GC noise gets ignored, +// which is worse than not having one. +if (growth > 50 && last.Bytes - first.Bytes > 32 * 1024 * 1024) +{ + invariants.Record(cycles, $"managed memory grew {growth:0}% ({(last.Bytes - first.Bytes) / 1024 / 1024}MB) after warmup"); +} + +// Every notification the server delivered should be raised exactly once - with one bounded exception. A +// handoff replaces the connection, and a frame already queued for the socket it replaces is lost: the server +// counted a write, but nobody was left to read it. So loss is tolerated up to the number of connection +// replacements, and anything beyond that is a real leak in the receive path. Over-raising is never tolerated: +// it would mean the collapse has stopped collapsing. +var raised = Interlocked.Read(ref events); +var lost = expectedEvents - raised; +var handoffs = server.HandoffRecycles; +Console.WriteLine($"unraised: {lost} (tolerance {handoffs}, one per connection replacement)"); + +if (raised > expectedEvents) +{ + invariants.Record(cycles, $"{raised} notifications raised for only {expectedEvents} delivered - the collapse is not collapsing"); +} +else if (lost > handoffs) +{ + invariants.Record(cycles, $"{lost} delivered notifications were never raised, with only {handoffs} connection replacements to explain them"); +} + +if (server.HandoffRecycles == 0) +{ + invariants.Record(cycles, "no handoff ever ran, so this run says nothing about the handoff path"); +} + +if (invariants.Violations.Count == 0) +{ + Console.WriteLine(); + Console.WriteLine("PASS: no invariant violated"); + return 0; +} + +Console.WriteLine(); +Console.Error.WriteLine($"FAIL: {invariants.Violations.Count} invariant violation(s)"); +foreach (var violation in invariants.Violations) Console.Error.WriteLine($" {violation}"); +return 1; + +static async Task SettlesAsync(Func condition, TimeSpan timeout) +{ + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (condition()) return true; + await Task.Delay(100); + } + + return condition(); +} diff --git a/toys/MaintenanceSoak/SoakServer.cs b/toys/MaintenanceSoak/SoakServer.cs new file mode 100644 index 000000000..54668de2e --- /dev/null +++ b/toys/MaintenanceSoak/SoakServer.cs @@ -0,0 +1,110 @@ +using System; +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Server; + +namespace StackExchange.Redis.MaintenanceSoak; + +/// +/// Hosts the in-process RESP server on a real TCP socket. +/// +/// +/// A real socket rather than the tunnel the test suite uses, deliberately: a soak is looking for what leaks +/// over thousands of cycles - sockets, pipes, buffers, event handlers - and the tunnel bypasses exactly the +/// machinery most likely to leak. It also means the connection has a genuine remote address, so the handoff's +/// endpoint-type derivation runs for real instead of resolving to "no address to classify". +/// +internal sealed class SoakServer : IAsyncDisposable +{ + private readonly Socket _listener; + private readonly CancellationTokenSource _shutdown = new(); + private readonly Task _accepting; + + public SoakServer(MemoryCacheRedisServer server, int port) + { + Server = server; + _listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + _listener.Bind(new IPEndPoint(IPAddress.Loopback, port)); + _listener.Listen(64); + EndPoint = (IPEndPoint)_listener.LocalEndPoint!; + _accepting = AcceptLoopAsync(); + } + + public MemoryCacheRedisServer Server { get; } + + public IPEndPoint EndPoint { get; } + + private async Task AcceptLoopAsync() + { + while (!_shutdown.IsCancellationRequested) + { + Socket socket; + try + { + socket = await _listener.AcceptAsync(_shutdown.Token); + } + catch (Exception) when (_shutdown.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + Console.WriteLine($"accept failed: {ex.Message}"); + continue; + } + + _ = ServeAsync(socket); + } + } + + private async Task ServeAsync(Socket socket) + { + try + { + socket.NoDelay = true; + using var stream = new NetworkStream(socket, ownsSocket: true); + + // one pipe each way, which is what RunClientAsync expects + var input = PipeReader.Create(stream); + var output = PipeWriter.Create(stream); + await Server.RunClientAsync(new DuplexPipe(input, output)); + } + catch (Exception ex) when (ex is System.IO.IOException or ObjectDisposedException or SocketException) + { + // an ordinary disconnect; the soak causes plenty of them on purpose + } + catch (Exception ex) + { + Console.WriteLine($"client faulted: {ex.GetType().Name}: {ex.Message}"); + } + } + + public async ValueTask DisposeAsync() + { + _shutdown.Cancel(); + try + { + _listener.Dispose(); + } + catch { } + + try + { + await _accepting; + } + catch { } + + Server.Dispose(); + _shutdown.Dispose(); + } + + private sealed class DuplexPipe(PipeReader input, PipeWriter output) : IDuplexPipe + { + public PipeReader Input => input; + public PipeWriter Output => output; + } +} From c733b1c08ecc933a7a3fa4d284b8334930b969da Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 13:34:47 +0100 Subject: [PATCH 13/36] Prove relaxed timeouts rescue a command, and attribute one that fails D4 was implemented and unit-tested but had no evidence of a relaxed window actually saving a command: four scenario runs against a real deployment produced zero command failures, which is the right product outcome and no use as proof. The fault injector's network_latency turned out to be the wrong instrument, and establishing that cost a cluster. It takes a bdb_id, which implies database scope, and does not have it: the response reports netem applied to whole node interfaces, so it delays the cluster's internal traffic and its DNS along with the client's. duration_seconds is accepted, echoed back, and does not revert - a 200ms injection across two of three nodes stayed applied, took the databases offline and made the cluster's own DNS zone unresolvable, since those nodes serve it. The environment had to be recreated. RedisServer.ResponseDelay does the job precisely instead, and the experiment is an A/B with the ambiguity removed: two multiplexers against one server, the same delay, and the notification delivered to one client only - relaxation is per-server-per-multiplexer, so the connection that was never told keeps its ordinary timeout. With a 200ms configured timeout and a 3s reply, simultaneously: told about the disruption: succeeded after 3000ms not told: RedisTimeoutException and a command timing out inside a window carries MaintenanceType = FailingOver. Two things the tests had to learn, both general rather than specific to this feature. Async timeouts are raised by the bridge heartbeat on roughly a one-second cadence rather than at the deadline, so a 600ms delay against a 200ms timeout succeeds - which is how the first control case passed when it should have failed. And attribution is read when the timeout is *raised*: a window that has already expired attributes nothing, even though its disruption caused the delay, which is worth revisiting since the natural reading of MaintenanceType is "was this caused by maintenance". Also ruled out: sequencing fail-then-succeed on one connection. A timed-out command disrupts it, and with a slow reply still configured the reconnect handshake cannot finish, so the second half fails in the backlog for an unrelated reason. --- .../MaintenanceRelaxationEvidenceTests.cs | 134 ++++++++++++++++++ .../RedisServer.Maintenance.cs | 27 ++++ 2 files changed, 161 insertions(+) create mode 100644 tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs new file mode 100644 index 000000000..293ced8f8 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Does relaxation actually save a command, and does a timeout inside a window say why? +/// +/// +/// The gap these close: everything about relaxed timeouts was implemented and unit-tested, but nothing had ever +/// observed one *rescue* a command - four scenario runs against a real deployment produced zero command +/// failures, which is the right product outcome and useless as evidence. +/// +/// The obvious way to force failures - the fault injector's network_latency - turned out to be the wrong +/// tool. It applies netem to a whole node's interface, so it delays the cluster's own traffic and its DNS along +/// with the client's; 200ms across two of three nodes took a working deployment offline, and its +/// duration_seconds did not revert. A per-reply delay in the fake is precise, instant, and cannot break +/// anything - and it makes the difference measurable rather than anecdotal. +/// +/// +public class MaintenanceRelaxationEvidenceTests(ITestOutputHelper log) +{ + // The delays have to clear the heartbeat, not just the timeout: async timeouts are raised by the bridge + // heartbeat sweep on roughly a one-second cadence, not at the instant the deadline passes. A 600ms delay + // against a 200ms timeout therefore completes *successfully*, which is how the first version of this test + // found its control case passing when it should have failed. + private const int NormalTimeoutMs = 200, RelaxedTimeoutSeconds = 8, SlowReplySeconds = 3; + + private static async Task ConnectAsync(InProcessTestServer server) + { + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + config.SyncTimeout = NormalTimeoutMs; + config.AsyncTimeout = NormalTimeoutMs; + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(RelaxedTimeoutSeconds); + + return await ConnectionMultiplexer.ConnectAsync(config); + } + + private static List OptedIn(InProcessTestServer server) + { + var found = new List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } + + [Fact] + public async Task ARelaxedWindowRescuesACommandThatWouldOtherwiseTimeOut() + { + // Two clients, one server, one delay - and the notification sent to only one of them. That is the whole + // experiment: identical conditions, simultaneously, differing only in whether a disruption was + // announced. Relaxation is per-server-per-multiplexer, so the client that was never told keeps its + // ordinary timeout. + // + // Deliberately not "fail, then succeed" on a single connection: a timed-out command disrupts the + // connection, and with a slow reply still configured the reconnect handshake cannot complete, so the + // second half then fails in the backlog for an entirely unrelated reason. That is what the first + // version of this test did. + using var server = new InProcessTestServer(log); + + await using var told = await ConnectAsync(server); + var toldClient = Assert.Single(OptedIn(server)); + + await using var untold = await ConnectAsync(server); + Assert.Equal(2, OptedIn(server).Count); + + await told.GetDatabase().PingAsync(); + await untold.GetDatabase().PingAsync(); // both healthy before anything is slowed + + // well past the normal timeout *and* the heartbeat, comfortably inside the relaxed one + server.ResponseDelay = TimeSpan.FromSeconds(SlowReplySeconds); + + // announced to one connection only + server.SendShardNotification(toldClient, MaintenanceNotificationKind.Migrating, timeSeconds: 20, shardIds: "[\"1\"]", sequenceId: 0); + var endpoint = ((IInternalConnectionMultiplexer)told).GetServerEndPoint(server.DefaultEndPoint); + Assert.True( + await Poll.UntilAsync(() => endpoint.IsMaintenanceRelaxed, timeoutMilliseconds: 10_000), + "the notification should have relaxed timeouts on the connection that received it"); + + var rescued = told.GetDatabase().PingAsync(); + var doomed = untold.GetDatabase().PingAsync(); + + var rtt = await rescued; + log.WriteLine($"told about the disruption: succeeded after {rtt.TotalMilliseconds:0}ms"); + Assert.True(rtt.TotalMilliseconds > NormalTimeoutMs, "the command should have taken longer than the normal timeout allows"); + + var failure = await Assert.ThrowsAnyAsync(() => doomed); + log.WriteLine($"not told: {failure.GetType().Name}"); + Assert.True(failure is RedisTimeoutException or TimeoutException, $"expected a timeout, got {failure}"); + } + + [Fact] + public async Task ATimeoutInsideAWindowSaysWhichEventCausedIt() + { + using var server = new InProcessTestServer(log); + await using var conn = await ConnectAsync(server); + { + var db = conn.GetDatabase(); + await db.PingAsync(); + + // A long window on purpose. The attribution is read when the timeout is *raised*, so a window that + // has already expired attributes nothing - even though its disruption is what caused the delay. + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 20, shardIds: "[\"7\"]", sequenceId: 0); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(await Poll.UntilAsync(() => endpoint.IsMaintenanceRelaxed, timeoutMilliseconds: 5_000)); + + // beyond even the relaxed timeout, so the command fails *during* an announced disruption + server.ResponseDelay = TimeSpan.FromSeconds(RelaxedTimeoutSeconds * 3); + + var failure = await Assert.ThrowsAnyAsync(() => db.PingAsync()); + log.WriteLine($"{failure.GetType().Name}: {failure.Message}"); + + // This is what the attribution is for: "the deployment was failing over" rather than "your query + // is slow". Nothing had ever observed it fire, because no real run ever failed a command. + var maintenanceType = failure switch + { + RedisTimeoutException timeout => timeout.MaintenanceType, + RedisConnectionException connection => connection.MaintenanceType, + _ => MaintenanceNotificationType.None, + }; + + log.WriteLine($"attributed to: {maintenanceType}"); + Assert.Equal(MaintenanceNotificationType.FailingOver, maintenanceType); + } + } +} diff --git a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs index 51530f5a9..7f3878309 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs @@ -56,6 +56,33 @@ public enum MaintenanceNotificationKind _ => throw new ArgumentOutOfRangeException(nameof(kind)), }; + /// + /// An artificial delay applied before each reply, to make command timeouts happen on demand. + /// + /// + /// This is how "does relaxation actually save a command?" gets tested. The obvious alternative - the + /// fault injector's network_latency action against a real cluster - turns out to be the wrong + /// tool: it applies netem to a whole *node's* interface, so it delays the cluster's internal traffic + /// and its DNS as well as the client's, and 200ms across two of three nodes was enough to take a + /// working deployment offline. Its duration_seconds did not revert either. A per-connection + /// delay here is precise, instant, and cannot break anything. + /// + /// Applies to *every* reply on the connection, including a handshake, so set it after connecting + /// unless a slow handshake is what is being tested. + /// + /// + public TimeSpan ResponseDelay { get; set; } + + /// + /// + /// Not an async method: the base signature takes the request by in, which async forbids. + /// + protected override ValueTask ClientPauseAsync(RedisClient client, in RedisRequest request) + { + var delay = ResponseDelay; + return delay > TimeSpan.Zero ? new ValueTask(Task.Delay(delay)) : default; + } + private int _maintenanceSequence; private int _maintenanceOptIns; private (MaintenanceNotificationKind Kind, long Sequence, string ShardIds)? _retainedCompletion; From 3e975df58a38a48fe9dfc162565589ecd27d213d Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 15:05:23 +0100 Subject: [PATCH 14/36] Measure how long a completion stays retained The last unmeasured property of the catch-up channel. Captures already show that a connection opting in after a shard-scoped event gets the event's completion replayed within ~17ms, most-recent-replaces, and that starters and MOVING are never retained. What nothing has established is whether that retention ages out - which matters because a completion replayed hours later would open a relaxed window for an event that finished long ago, and would be worth an age guard if the server has none. Fires one failover, then probes on a ladder up to whatever horizon is asked for via SER_FI_RETENTION_AGE_MINUTES; without that it skips, since it is a measurement that spends hours waiting rather than something an ordinary run of the tier should carry. Three parts are load-bearing rather than incidental: - The observable is the endpoint's relaxed state, not ServerMaintenanceEvent. A retained completion arrives inside ConnectAsync, so a handler attached after connecting has already missed it; the window it opened is still there to read. The fake's retention test takes the same approach for the same reason. - Two probes per rung. If the first sees the replay and the second does not, the server clears the retained item on delivery, and every later rung is measuring an empty channel rather than an expired one - a confound that is invisible with one probe per rung and looks exactly like an early expiry. Measured today: both see it, so retention survives delivery. - Progress is written to a file, flushed per line, because ITestOutputHelper is buffered until the test ends: over three hours, a run in progress and a wedged run are otherwise indistinguishable. A probe that cannot connect is recorded as inconclusive rather than as a miss, so a cluster whose lease expires mid-run cannot masquerade as an expiry. --- .../README.md | 24 ++ .../RetentionAgeScenarioTests.cs | 291 ++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index 2be42b3a7..7f3c33f28 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -20,6 +20,30 @@ it already holds the cluster credentials (`env_output.json`), the CA certificate has to be hand-carried into the test run. `FAULT_INJECTION_API_URL` overrides the injector URL (default `http://127.0.0.1:20324`). +## The one test that is opt-in even here + +`RetentionAgeScenarioTests` measures how long the server retains a completion for replay to a +newly-opted-in connection - the last unmeasured property of the catch-up channel. It fires one failover and +then probes on a ladder (1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 180, 240 minutes), so it runs for as long as +you let it and skips unless you ask for it: + +```bash +export SER_FI_RETENTION_AGE_MINUTES=180 # trims the ladder; absent means skip +export SER_FI_RETENTION_AGE_LOG=/tmp/age.log # optional; defaults under the temp directory +``` + +Two details that are not incidental: + +- **Progress is written to a file, flushed per line.** `ITestOutputHelper` is buffered until the test ends, so + over three hours a run in progress and a run that has wedged look identical through the normal channel. +- **Two probes per rung.** If the first sees the replay and the second does not, the server clears the retained + item on delivery - in which case later rungs are measuring an empty channel rather than an expired one. That + confound is invisible with one probe per rung and looks exactly like an early expiry. (Measured 2026-09-02: + both probes see it, so retention is not consumed on delivery.) +- A probe that cannot connect is recorded as **inconclusive, not as a miss**: the run outlives its cluster's + lease easily, and counting a dead environment as "no replay" would report an expiry at whatever minute the + cluster went away. + ## Three states, deliberately distinct | state | behaviour | diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs new file mode 100644 index 000000000..4bb7b4c7a --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs @@ -0,0 +1,291 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// How long a retained completion stays retained - the last unmeasured property of the catch-up channel. +/// +/// +/// What is already known, from captures: a connection that opts in *after* a shard-scoped event gets the +/// event's **completion** replayed (MIGRATED, FAILED_OVER; never a starter, never +/// MOVING), delivered within ~17ms of the opt-in being accepted, most-recent-replaces. What is not +/// known is whether that retention ever ages out. It matters because a client is entitled to act on what it +/// receives: a completion replayed hours later would open a relaxed window for an event that finished long +/// ago, which is harmless-but-wasteful for us and would be worth an age guard if the server does not have one. +/// +/// Written as a measurement rather than a pass/fail: the schedule is a ladder, every probe is logged, and the +/// only hard assertions are the ones that say the measurement itself is sound. Opt in with +/// SER_FI_RETENTION_AGE_MINUTES=<minutes>; without it this skips, because it fires one failover +/// and then spends the rest of its time waiting, which has no place in an ordinary run of the tier. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "retention-age")] +public class RetentionAgeScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string HorizonVariable = "SER_FI_RETENTION_AGE_MINUTES"; + private const string ProgressVariable = "SER_FI_RETENTION_AGE_LOG"; + + /// + /// Progress is written to a file as it happens, as well as to the test output. + /// + /// + /// is buffered until the test finishes, and this test runs for hours - so + /// through the only channel a test normally has, a run in progress and a run that has wedged look + /// identical. The file is flushed per line, so the ladder can be read while it is still being climbed. + /// + private static string ProgressPath => + Environment.GetEnvironmentVariable(ProgressVariable) is { Length: > 0 } configured + ? configured + : Path.Combine(Path.GetTempPath(), "ser-retention-age.log"); + + /// Probe ages, in minutes since the completion; trimmed to whatever horizon was asked for. + /// + /// Dense early and sparse later: an expiry at 30 seconds and an expiry at two hours are both plausible, and + /// a geometric ladder pins either to within a factor of two without spending the whole cluster lease. + /// + private static readonly int[] LadderMinutes = [1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 180, 240]; + + [Fact] + public async Task HowLongIsACompletionRetained() + { + fixture.RequireAvailable(); + var horizon = ReadHorizon(); + if (horizon is null) + { + Assert.Skip( + $"set {HorizonVariable}= to run this; it fires one failover and then probes for that " + + "long, so it is opt-in even within this tier"); + } + + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + + using var progress = new StreamWriter(ProgressPath, append: true) { AutoFlush = true }; + _progress = progress; + Note($"--- retention age, horizon {horizon} minutes, started {DateTime.UtcNow:u} ---"); + Note($"provisioned {database}"); + + // The witness stays connected for the whole run, for one reason: retention is most-recent-replaces, so + // any *further* event on this database resets the age we are measuring. If one arrives, the ladder from + // that point on is measuring the new event, and the log has to show that rather than hide it. + var clock = Stopwatch.StartNew(); + var witnessed = new List<(TimeSpan At, PushMaintenanceEvent Push)>(); + await using var witness = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + witness.ServerMaintenanceEvent += (_, e) => + { + if (e is not PushMaintenanceEvent push) return; + lock (witnessed) witnessed.Add((clock.Elapsed, push)); + Note($" witness +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + }; + + await witness.GetDatabase().StringSetAsync("fi-retention-age", "before"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "failover", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not run 'failover' against bdb {database.BdbId}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + Note($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports the failover finished"); + + // A completion is what gets retained, so the clock we care about starts when one arrives - not when the + // scenario was fired, and not when the injector called it done. + var completed = await Poll.UntilAsync( + () => + { + lock (witnessed) return witnessed.Any(w => IsCompletion(w.Push.NotificationType)); + }, + timeoutMilliseconds: 120_000); + + if (!completed) + { + lock (witnessed) + { + var seen = witnessed.Count == 0 ? "nothing" : string.Join(", ", witnessed.Select(w => w.Push.NotificationType)); + Assert.Skip($"no completion was announced within 120s (saw {seen}), so there is nothing whose retention could be measured"); + } + } + + TimeSpan completionAt; + long completionSequence; + MaintenanceNotificationType completionType; + lock (witnessed) + { + var completion = witnessed.First(w => IsCompletion(w.Push.NotificationType)); + completionAt = completion.At; + completionSequence = completion.Push.SequenceId; + completionType = completion.Push.NotificationType; + } + + Note($"completion to measure: {completionType} seq={completionSequence} at +{completionAt.TotalSeconds:0.0}s"); + + var results = new List(); + foreach (var minutes in LadderMinutes.Where(m => m <= horizon)) + { + var due = completionAt + TimeSpan.FromMinutes(minutes); + var wait = due - clock.Elapsed; + if (wait > TimeSpan.Zero) await Task.Delay(wait, cancellationToken); + + // Two connections per rung, back to back. If the first sees the replay and the second does not, the + // server clears the retained item once it has been delivered - in which case every later rung is + // measuring an empty channel rather than an expired one, and the ladder means nothing. That + // confound is invisible with one connection per rung, and it would look exactly like an early + // expiry. + var first = await ProbeAsync(database, minutes, "a", cancellationToken); + var second = await ProbeAsync(database, minutes, "b", cancellationToken); + results.Add(first); + results.Add(second); + + if (first.Replayed && !second.Replayed) + { + Note( + $" !! at {minutes}m the first probe saw the replay and the second did not: retention looks " + + "consumed on delivery, so rungs beyond this one cannot be read as ages"); + } + } + + Note(string.Empty); + Note("age probe replayed type notes"); + foreach (var probe in results) + { + var verdict = probe.Conclusive ? (probe.Replayed ? "yes" : "no") : "n/a"; + Note($"{probe.Minutes,4}m {probe.Label,-5} {verdict,-8} {probe.Type,-12} {probe.Notes}"); + } + + lock (witnessed) + { + var later = witnessed.Where(w => w.At > completionAt).ToList(); + if (later.Count != 0) + { + Note( + " !! further events arrived after the completion, so the retained item was replaced and the " + + $"ages above are relative to the wrong event: {string.Join(", ", later.Select(w => $"{w.Push.NotificationType}@+{w.At.TotalSeconds:0}s"))}"); + } + } + + // The measurement is the output; these two assertions exist so that a run which proves nothing says so + // instead of being read as "retention expires immediately". + Assert.NotEmpty(results); + Assert.True(results[0].Conclusive, $"the first probe could not connect: {results[0].Notes}"); + Assert.True( + results[0].Replayed, + $"the first probe ({results[0].Minutes}m after a {completionType}) saw no replay at all, so this run " + + "measured nothing - either retention is shorter than the first rung, or the opt-in is not being honoured"); + + // Once it stops being replayed it must stay stopped. An age guard is monotone; a completion that + // reappears after a gap would mean something much stranger than expiry, and is worth failing on. + var byRung = results.Where(r => r is { Label: "a", Conclusive: true }).ToList(); + if (byRung.Count == 0) + { + Assert.Fail("every probe failed to connect, so nothing was measured"); + } + + var firstMiss = byRung.FindIndex(r => !r.Replayed); + if (firstMiss >= 0) + { + var after = byRung.Skip(firstMiss).Where(r => r.Replayed).ToList(); + Assert.True( + after.Count == 0, + $"the replay stopped at {byRung[firstMiss].Minutes}m and then came back at " + + $"{string.Join(", ", after.Select(r => r.Minutes + "m"))}, which no expiry rule explains"); + Note($"=> retention lapsed between {(firstMiss == 0 ? 0 : byRung[firstMiss - 1].Minutes)}m and {byRung[firstMiss].Minutes}m"); + } + else + { + Note($"=> still replayed at {byRung[^1].Minutes}m: retention outlasts the horizon asked for"); + } + } + + /// + /// One fresh connection, and what the server told it on the way in. + /// + /// + /// The observable is the endpoint's relaxed state, not the ServerMaintenanceEvent, and that is not a + /// convenience: a retained completion arrives within ~17ms of the opt-in being accepted, which is *inside* + /// ConnectAsync, so a handler attached after connecting has already missed it. The relaxed window it + /// opened is still there to be read. + /// + private async Task ProbeAsync(ProvisionedDatabase database, int minutes, string label, CancellationToken cancellationToken) + { + try + { + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + var relaxed = endpoint.IsMaintenanceRelaxed; + var type = endpoint.ActiveMaintenanceType; + + // a live event arriving *during* the probe would also relax the window, so record anything that + // shows up while we are here; the witness sees it too, and the two together tell them apart + var live = new List(); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) lock (live) live.Add(push.NotificationType); + }; + await conn.GetDatabase().PingAsync(); + await Task.Delay(2000, cancellationToken); + + string notes; + lock (live) + { + notes = live.Count == 0 ? string.Empty : $"also received live: {string.Join(", ", live)}"; + } + + Note($" probe {minutes}m/{label}: relaxed={relaxed} type={type} {notes}"); + return new Probe(minutes, label, relaxed, type, notes, Conclusive: true); + } + catch (Exception ex) + { + // Inconclusive, emphatically not "no replay": this runs for hours against a cluster with a lease on + // it, and a probe that cannot connect at all says nothing about retention. Counting it as a miss + // would report an expiry at whatever minute the environment went away. + Note($" probe {minutes}m/{label}: failed to connect: {ex.GetType().Name}: {ex.Message}"); + return new Probe(minutes, label, false, MaintenanceNotificationType.None, $"connect failed: {ex.GetType().Name}", Conclusive: false); + } + } + + private StreamWriter? _progress; + + private void Note(string message) + { + log.WriteLine(message); + _progress?.WriteLine(message.Length == 0 ? message : $"{DateTime.UtcNow:HH:mm:ss} {message}"); + } + + private static bool IsCompletion(MaintenanceNotificationType type) + => type is MaintenanceNotificationType.Migrated or MaintenanceNotificationType.FailedOver; + + private static int? ReadHorizon() + { + var raw = Environment.GetEnvironmentVariable(HorizonVariable); + return int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var minutes) && minutes > 0 + ? minutes + : null; + } + + private readonly record struct Probe( + int Minutes, + string Label, + bool Replayed, + MaintenanceNotificationType Type, + string Notes, + bool Conclusive); +} From d033d9c7727c7ddc345df53219cb93b8aebbb544 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 15:06:36 +0100 Subject: [PATCH 15/36] Key the handoff count on distinct MOVING sequences, not one overall HandoffBeatsTheServerToTheClose asserted exactly one recycle. On the data_movement_conn_drop/maintenance_mode case it saw two, and the client was right both times: that trigger reports automatically_clean_mm: false, so the node stays in maintenance mode after the effect lands and coming back out of it moves the shard again - MOVING seq=2 at +14.9s and seq=3 at +97.0s, 82s apart, one recycle each. The invariant is one recycle per distinct notification, so the test now counts the MOVING sequence numbers it observed and compares against HandoffRecycles. That still catches the loop this assertion was written for - a server re-sends MOVING to a connection that opts in mid-window, and since the handoff replaces the connection, acting on the repeat produces another one - because a replay repeats the sequence number. --- .../MovingHandoffScenarioTests.cs | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs index 2587af7f2..a5c6acbee 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs @@ -71,9 +71,15 @@ void Note(string what) Note($"disconnect: {e.FailureType}"); }; conn.ConnectionRestored += (_, _) => Note("reconnected"); + var movingSequences = new HashSet(); conn.ServerMaintenanceEvent += (_, e) => { - if (e is PushMaintenanceEvent push) Note($"{push.NotificationType} seq={push.SequenceId} time={push.Time?.TotalSeconds.ToString() ?? "-"}"); + if (e is not PushMaintenanceEvent push) return; + Note($"{push.NotificationType} seq={push.SequenceId} time={push.Time?.TotalSeconds.ToString() ?? "-"}"); + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + lock (movingSequences) movingSequences.Add(push.SequenceId); + } }; clock.Restart(); @@ -108,12 +114,21 @@ void Note(string what) // invisible, which is worth knowing in its own right and is why HandoffRecycles exists. } - // Exactly one handoff per notification. More than one is the feedback loop this test found on its first - // live run: a server re-sends MOVING to a connection that opts in while the window is still open, and - // since the handoff replaces the connection, acting on the repeat produces another one - twelve - // recycles from a single event. The per-server sequence dedup now gates it, and this is the assertion - // that would catch a regression. - Assert.Equal(1, endpoint.HandoffRecycles); + // Exactly one handoff per distinct notification - not one overall, which is what this asserted until a + // live run proved the scenario can announce twice. The `maintenance_mode` trigger reports + // `automatically_clean_mm: false`, so the node stays in maintenance mode after the effect lands and + // coming back out of it moves the shard again: MOVING seq=2 at +14.9s and seq=3 at +97.0s, one recycle + // each, and the client was right both times. + // + // What the count still catches is the feedback loop this test found on its first live run: a server + // re-sends MOVING to a connection that opts in while the window is still open, and since the handoff + // replaces the connection, acting on the repeat produces another one - twelve recycles from *one* + // event. A replay repeats the sequence number, so keying on the sequence keeps that sharp while + // letting a genuine second event through. + int announced; + lock (movingSequences) announced = movingSequences.Count; + Assert.True(announced > 0, "no MOVING was announced, so this test would prove nothing"); + Assert.Equal(announced, endpoint.HandoffRecycles); Assert.NotNull(endpoint.LastHandoffOutcome); // Which outcome depends on whether the server named a replacement, and it only does that when we asked From e3ad81c5f386165ff31c65647dbac9fbec34a5ec Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 17:09:53 +0100 Subject: [PATCH 16/36] A catch-up completion opens no relaxed window Measured on a live deployment: Redis Enterprise retains the most recent shard-scoped completion and replays it to whoever opts in next, and that retention has no age limit worth relying on - the same FAILED_OVER was still being replayed to fresh connections 90 minutes after the failover, to two connections per attempt, so it is not consumed on delivery either. Completions carry no time field (starters do), so nothing in the frame distinguishes "just happened" from "happened this morning". The only signal available is when it arrived: during the opt-in, or on a live connection. Before this, every new connection to a database that had ever failed over began life with the full post-event tail of relaxed timeouts and attributed any timeout inside it to maintenance that was long over. So a completion arriving before the bridge reports established gets no tail. It declines to *open* a window rather than closing one, which matters: the window belongs to the ServerEndPoint and is shared by both bridges, so a subscription bridge reconnecting mid-disruption - handed the retained completion of some earlier event on the way in - must not cancel the window the live notification opened on the interactive bridge. Also adds the log line for a received notification (event id 121), which did not exist: Trace is [Conditional("VERBOSE")], so a notification was invisible in an ordinary deployment, including to the log-based verification our own documentation recommends. It names the catch-up case, so "why is my new connection relaxed?" has an answer in the log. That log is also what the retention tests now assert on. They used the relaxed window as their observable - the event collector attaches after ConnectAsync and has already missed a retained frame - and with no window to look at, the observable has to be something attachable before connecting. Two of them would otherwise have become vacuous, including the one guarding that MOVING is never retained. --- docs/ServerMaintenanceEvent.md | 23 +++ src/StackExchange.Redis/LoggerExtensions.cs | 7 + .../PhysicalConnection.Maintenance.cs | 16 +- .../ServerEndPoint.Maintenance.cs | 24 ++- .../MaintenanceNotificationTests.cs | 137 ++++++++++++++---- .../MaintenanceRelaxationTests.cs | 46 ++++++ 6 files changed, 220 insertions(+), 33 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 6995a7e5f..9e1b7aded 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -168,6 +168,17 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. +### Notifications that arrive as you connect + +Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_OVER` - and replays it to each connection that opts in, so a client that connects after a disruption still learns that it happened. Measured behaviour, worth knowing if you handle these events yourself: + +* Only completions are replayed. Starters (`MIGRATING`, `FAILING_OVER`) are not, and neither is `MOVING` - so a replay can never demand that you move. +* One item, most-recent-replaces; there is no queue. +* It arrives within milliseconds of the opt-in being accepted, which is *during* connection establishment - so an event handler attached after `ConnectAsync` returns will usually not see it. +* **It can be very old.** The same `FAILED_OVER` was still being replayed to fresh connections 90 minutes after the failover, with no expiry observed, and a completion carries no time field - so nothing in the notification says how old it is. + +Because of that last point, a completion that arrives while the connection is still being established does **not** relax timeouts: it is history, not news. A completion that arrives on a live connection does, as the table above says. If you act on these events yourself, treat one that arrives at connection time as "this happened at some point", not "this is happening". + Note that a deliberate handoff appears as a `ConnectionFailed` event with `FailureType == ConnectionFailureType.MaintenanceHandoff`. That is expected during planned maintenance and does not indicate a fault; if you alert on `ConnectionFailed`, filter it out. ## Watching the events @@ -206,6 +217,8 @@ Three settings control the relaxed window, all in seconds: | `maintRelaxedWindowMax` | 3x the relaxed timeout | the longest a single window may last, in case a closing notification never arrives | | `maintPostEventRelaxed` | 2x the relaxed timeout | how long timeouts stay relaxed *after* the disruption ends | +The tail applies to a completion that arrives on a live connection. A completion replayed as you connect gets no tail at all, for the reasons above. + The announced duration is clamped rather than honoured literally. Windows as short as two seconds have been observed in practice, which is not long enough to cover a client reconnecting, and a client that trusted the announced value would stop being patient exactly when it mattered. The tail exists for the same reason in reverse: after a handoff, servers and other clients are still settling. If a command does time out during a window, the exception carries the reason: `RedisTimeoutException.MaintenanceType` (and the same property on `RedisConnectionException`) names the notification that was in effect, which distinguishes "the deployment was moving" from "this query is slow". @@ -230,6 +243,16 @@ or, when the server declines, the reason it gave: 10.0.0.1:6379: Maintenance notifications refused (ERR maintenance notifications are disabled on this server) ``` +Received notifications are logged too, which is the quickest way to answer "did anything actually arrive?" - and, when a new connection is unexpectedly patient about timeouts, "was that a replay?": + +``` +10.0.0.1:6379: Maintenance notification: FailingOver seq=41 +10.0.0.1:6379: Maintenance notification: FailedOver seq=42 +10.0.0.2:6379: Maintenance notification: FailedOver seq=42 (catch-up) +``` + +The last line is the retained copy described above, delivered to a connection that opted in afterwards. + A handoff is reported the same way, which is worth knowing because it replaces connections: ``` diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 27026c28e..ea21d0708 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -3,6 +3,7 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using StackExchange.Redis.Maintenance; namespace StackExchange.Redis; @@ -784,6 +785,12 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "{Server}: Maintenance notifications refused ({Reason})")] internal static partial void LogInformationMaintenanceNotificationsRefused(this ILogger logger, ServerEndPointLogValue server, string reason); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 121, + Message = "{Server}: Maintenance notification: {Type} seq={Sequence}{CatchUp}")] + internal static partial void LogInformationMaintenanceNotificationReceived(this ILogger logger, ServerEndPointLogValue server, MaintenanceNotificationType type, long sequence, string catchUp); + [LoggerMessage( Level = LogLevel.Information, EventId = 119, diff --git a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs index 03e74f875..670e1382e 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs @@ -124,10 +124,24 @@ private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, P Trace($"maintenance notification: {raw}"); OnDetailLog($"maintenance notification: {raw}"); + // A notification that arrives before the bridge reports established is the server's *catch-up* copy: + // it retains the completion of a shard-scoped event and replays it to whoever opts in next, with no + // measured age limit (the same FAILED_OVER came back 90 minutes later). Distinguishing the two matters + // for the completions, which otherwise relax timeouts on a brand-new connection for an event that + // finished long ago; the starters are unaffected, since nothing retains them. + var isCatchUp = BridgeCouldBeNull?.IsConnected != true; + // relax before reporting: the event handler is consumer code, and the window should already be open // by the time anyone sees the notification that opened it if (server is not null) { + // Logged, not merely traced: Trace is [Conditional("VERBOSE")], so until now a received + // notification was invisible in an ordinary deployment - including to the log-based verification + // our own documentation recommends. This is the line that answers "why is my new connection + // relaxed?", so it names the catch-up case explicitly. + muxer.Logger?.LogInformationMaintenanceNotificationReceived( + new(server), type, sequenceId ?? -1, isCatchUp ? " (catch-up)" : string.Empty); + if (IsWindowOpening(type)) { var isNew = server.OnMaintenanceWindowOpened(type, sequenceId, time); @@ -148,7 +162,7 @@ private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, P } else if (IsWindowClosing(type)) { - server.OnMaintenanceWindowClosed(type, sequenceId); + server.OnMaintenanceWindowClosed(type, sequenceId, isCatchUp); // ...and if slots moved away from us, learn the new topology rather than waiting to be told // by a -MOVED. Scoped and jittered inside OnSlotsMigratedAway; see its remarks for why this diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index c23030cb1..8fb899b71 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -236,10 +236,32 @@ internal bool OnMaintenanceWindowOpened(MaintenanceNotificationType type, long? /// previously said about duration is stale - but the tail still applies, because completion is when every /// other client that received the same notification re-engages. /// - internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? sequenceId) + /// Which completion this is. + /// The server's sequence number, for repeat detection. + /// + /// Whether this arrived as part of establishing the connection rather than on a live one - in which case + /// it is the server's retained copy of an event that has already finished, and gets no tail. + /// + internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? sequenceId, bool isCatchUp) { if (!TryClaimSequenceId(type, sequenceId)) return; + // A completion delivered while we were still connecting is the server's catch-up channel, and + // measurement says that channel has no age limit: the same FAILED_OVER was replayed to fresh + // connections 90 minutes after the failover, and completions carry no time field, so nothing in the + // frame distinguishes "just happened" from "happened this morning". Without this, every new + // connection to a database that had ever failed over began life with the full post-event tail of + // relaxed timeouts, and reported any timeout inside it as caused by maintenance that was long over. + // + // Note this declines to *open* a window rather than closing one. Relaxation belongs to the + // ServerEndPoint and is shared by both bridges, so a catch-up arriving on a reconnecting subscription + // bridge must not cancel a window that a live notification opened on the established interactive one. + if (isCatchUp) + { + Multiplexer.Trace($"{type}: catch-up copy of a finished event; no relaxation", ToString()); + return; + } + Volatile.Write(ref _relaxedType, (int)type); var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; if (tail <= TimeSpan.Zero) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index f8e904b63..9608efa36 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using StackExchange.Redis.Maintenance; using Xunit; using static StackExchange.Redis.Server.RedisServer; @@ -481,7 +482,8 @@ public class Retention(ITestOutputHelper log) { private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( ITestOutputHelper log, - Action beforeConnect) + Action beforeConnect, + NotificationLog? notifications = null) { var server = new InProcessTestServer(log); beforeConnect(server); // the event happens *before* anybody connects: that is the whole point @@ -489,28 +491,81 @@ public class Retention(ITestOutputHelper log) var config = server.GetClientConfig(defaultOnly: true); config.Protocol = RedisProtocol.Resp3; config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + config.LoggerFactory = notifications; var conn = await ConnectionMultiplexer.ConnectAsync(config); return (server, conn, new EventCollector(conn)); } + /// + /// Captures the library's own "maintenance notification" log lines. + /// + /// + /// The only observable that works here. A retained frame arrives while the connection is still + /// handshaking, so a handler attached after ConnectAsync has already missed it - and the relaxed + /// window, which these tests used to key on, is deliberately *not* opened for a catch-up any more. A + /// logger can be attached through the configuration before connecting, so it sees everything, and it + /// asserts on the notification itself rather than on a side-effect of it. + /// + private sealed class NotificationLog : ILoggerFactory, ILogger + { + private readonly List _received = []; + + public IReadOnlyList Received + { + get { lock (_received) return _received.ToArray(); } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + if (message.Contains("Maintenance notification:", StringComparison.Ordinal)) + { + lock (_received) _received.Add(message); + } + } + + public void Dispose() { } + } + [Theory] [InlineData(MaintenanceNotificationKind.Migrated)] [InlineData(MaintenanceNotificationKind.FailedOver)] public async Task RetainedCompletionIsReplayedToANewConnection(MaintenanceNotificationKind kind) { - var (server, conn, events) = await ConnectAsync(log, s => - s.SendShardNotification(null, kind, timeSeconds: null, shardIds: "[\"27\"]", sequenceId: 5)); + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => s.SendShardNotification(null, kind, timeSeconds: null, shardIds: "[\"27\"]", sequenceId: 5), + notifications); using (server) await using (conn) { - // the collector is attached *after* connecting, so this asserts the frame was handled rather - // than that the event fired - which is why the relaxed window is the observable here + // The frame was received and understood - asserted from the library's own log, because the + // event collector is attached after connecting and has therefore already missed it. + var received = Assert.Single(notifications.Received); + log.WriteLine(received); + Assert.Contains(MaintenanceNotificationTypeFor(kind).ToString(), received); + Assert.Contains("seq=5", received); + Assert.Contains("(catch-up)", received); + + // ...and it did *not* relax anything. Measured on a live deployment: the same completion is + // replayed to fresh connections at least 90 minutes after the event, and completions carry no + // time field, so relaxing here would mean every new connection to a database that had ever + // failed over began life patient about timeouts, and attributing any of them to maintenance + // that was long finished. var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); - Assert.True(endpoint.IsMaintenanceRelaxed, "the replayed completion should have opened the post-event tail"); - Assert.Equal(MaintenanceNotificationTypeFor(kind), endpoint.ActiveMaintenanceType); - log.WriteLine($"{kind} replayed; relaxed = {endpoint.IsMaintenanceRelaxed}"); + Assert.False(endpoint.IsMaintenanceRelaxed, "a catch-up completion should not open the post-event tail"); + Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); // ...and it must not have disturbed the handshake it arrived in the middle of, which is the // other half of what this test is for: a push frame interleaved with our own handshake @@ -540,22 +595,31 @@ public async Task StartersAndSlotScopedEventsAreNotRetained(MaintenanceNotificat // disruption ended", never "one is starting". Nothing that demands action is replayed, so a // reconnecting client cannot be told to move by a stale frame - which is what makes the MOVING // handoff safe from replay by construction rather than by a guard. - var (server, conn, events) = await ConnectAsync(log, s => - { - if (kind == MaintenanceNotificationKind.SlotMigrated) + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => { - s.SendSlotMigrations(null, kind, [("127.0.0.1:7000", "127.0.0.1:7001", "50-60")], sequenceId: 5); - } - else - { - s.SendShardNotification(null, kind, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); - } - }); + if (kind == MaintenanceNotificationKind.SlotMigrated) + { + s.SendSlotMigrations(null, kind, [("127.0.0.1:7000", "127.0.0.1:7001", "50-60")], sequenceId: 5); + } + else + { + s.SendShardNotification(null, kind, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + } + }, + notifications); using (server) await using (conn) { await events.AssertNoneAsync(); + + // Asserted from the log rather than from the relaxed window: a catch-up no longer relaxes + // anything, so "nothing relaxed" would now be true whether the frame was retained or not, and + // this test would pass without exercising the property it exists for. + Assert.Empty(notifications.Received); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); Assert.False(endpoint.IsMaintenanceRelaxed, $"{kind} must not be retained, so nothing should have relaxed"); } @@ -566,18 +630,24 @@ public async Task RetentionReplacesRatherThanAccumulates() { // "most recent completion", not a queue: a connection sees at most one of these however many // events went past, which is why the client side needs no window or ring for the catch-up case - var (server, conn, events) = await ConnectAsync(log, s => - { - s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"1\"]", sequenceId: 5); - s.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, null, "[\"2\"]", sequenceId: 6); - }); + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => + { + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"1\"]", sequenceId: 5); + s.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, null, "[\"2\"]", sequenceId: 6); + }, + notifications); using (server) await using (conn) { - var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); - Assert.True(endpoint.IsMaintenanceRelaxed); - Assert.Equal(MaintenanceNotificationType.FailedOver, endpoint.ActiveMaintenanceType); // the later one + // one replay, and it is the later event - not both, and not the earlier one + var received = Assert.Single(notifications.Received); + log.WriteLine(received); + Assert.Contains(nameof(MaintenanceNotificationType.FailedOver), received); + Assert.Contains("seq=6", received); GC.KeepAlive(events); } } @@ -587,16 +657,21 @@ public async Task RetentionCanBeTurnedOff() { // not every deployment retains, and a test of the no-retention case should not have to reason // about which notification kinds happen to be retained - var (server, conn, events) = await ConnectAsync(log, s => - { - s.RetainCompletions = false; - s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"27\"]", sequenceId: 5); - }); + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => + { + s.RetainCompletions = false; + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"27\"]", sequenceId: 5); + }, + notifications); using (server) await using (conn) { await events.AssertNoneAsync(); + Assert.Empty(notifications.Received); // nothing arrived at all, which is the point Assert.False(((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).IsMaintenanceRelaxed); } } diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs index c44296df0..e4a5fbd6e 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs @@ -111,6 +111,52 @@ public async Task ClosingNotificationLeavesThePostEventTail() } } + [Fact] + public async Task ACatchUpCompletionOpensNoTail() + { + // Measured on a live deployment: a completion is retained and replayed to whoever opts in next, with + // no age limit found (the same FAILED_OVER came back at least 90 minutes later), and completions carry + // no time field - so nothing in the frame says how old it is. Relaxing on one would mean every new + // connection to a database that had ever failed over started life patient about timeouts, and + // reporting any that expired as caused by maintenance long since finished. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + endpoint.OnMaintenanceWindowClosed(MaintenanceNotificationType.FailedOver, sequenceId: 41, isCatchUp: true); + + Assert.False(endpoint.IsMaintenanceRelaxed, "a catch-up completion should not relax anything"); + Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + } + } + + [Fact] + public async Task ACatchUpCompletionDoesNotCancelALiveWindow() + { + // The reason the catch-up path declines to *open* a window rather than closing one. Relaxation belongs + // to the ServerEndPoint and is shared by both bridges, so a subscription bridge that reconnects + // mid-disruption - and is handed the retained completion of some *earlier* event on the way in - must + // not cancel the window the live notification opened on the interactive bridge. Driven through the + // internals because the transport-level version of this needs one bridge to reconnect while another + // stays up, which is a lot of scaffolding to assert a state-machine rule. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 60); + Assert.True(await UntilRelaxedAsync(endpoint, true), "the live notification should have opened a window"); + + // an unrelated, older completion arrives as catch-up on a reconnecting bridge + endpoint.OnMaintenanceWindowClosed(MaintenanceNotificationType.Migrated, sequenceId: 7, isCatchUp: true); + + Assert.True(endpoint.IsMaintenanceRelaxed, "the live window should have survived the catch-up"); + Assert.Equal(MaintenanceNotificationType.FailingOver, endpoint.ActiveMaintenanceType); + } + } + [Fact] public async Task ClosingNotificationEndsItWhenThereIsNoTail() { From 75fa3e52f0c08e9844dc0aadc5bb1ecf21ffe62f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 17:11:51 +0100 Subject: [PATCH 17/36] Read the retention probe from the log, not the relaxed window Direct consequence of the previous commit: the probe detected a replay by reading IsMaintenanceRelaxed, which a catch-up completion deliberately no longer sets. Left alone, every rung would have reported "not replayed" - which reads exactly like an expiry at the first rung, i.e. the harness would have confidently reported the opposite of what it measured this afternoon. The log line added for the same reason is the right observable anyway: it is attachable before connecting, so unlike the event it cannot be missed, and it states the notification rather than a side-effect of it. The endpoint's window is still read, but only to note when something arrived *live* during a probe - in which case that rung is measuring the live event, not the retained one. --- .../RetentionAgeScenarioTests.cs | 80 +++++++++++++++++-- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs index 4bb7b4c7a..1d728b129 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; +using Microsoft.Extensions.Logging; using System.Threading; using System.Threading.Tasks; using StackExchange.Redis.Maintenance; @@ -219,19 +220,36 @@ await fixture.Injector.RunActionAsync( /// One fresh connection, and what the server told it on the way in. /// /// - /// The observable is the endpoint's relaxed state, not the ServerMaintenanceEvent, and that is not a + /// The observable is the client's own log, not the ServerMaintenanceEvent, and that is not a /// convenience: a retained completion arrives within ~17ms of the opt-in being accepted, which is *inside* - /// ConnectAsync, so a handler attached after connecting has already missed it. The relaxed window it - /// opened is still there to be read. + /// ConnectAsync, so a handler attached after connecting has already missed it. A logger can be + /// attached through the configuration beforehand. + /// + /// It used to read the endpoint's relaxed window instead, which was simpler and is no longer true: since a + /// catch-up completion is history rather than news, it deliberately opens no window - a change this + /// measurement is what prompted. Reading the window would now report "not replayed" at every rung, which + /// would look exactly like an expiry at the first one. + /// /// private async Task ProbeAsync(ProvisionedDatabase database, int minutes, string label, CancellationToken cancellationToken) { try { - await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var options = database.GetClientConfig(); + var notifications = new NotificationLog(); + options.LoggerFactory = notifications; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(options); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); - var relaxed = endpoint.IsMaintenanceRelaxed; - var type = endpoint.ActiveMaintenanceType; + var received = notifications.Received; + var relaxed = received.Count != 0; + var type = received.Count == 0 ? MaintenanceNotificationType.None : TypeOf(received[0]); + if (endpoint.IsMaintenanceRelaxed) + { + // not expected on a fresh connection any more; if it happens, something arrived *live* while + // we were connecting, and the rung is measuring that instead + Note($" probe {minutes}m/{label}: note - the window is open ({endpoint.ActiveMaintenanceType}), so a live event may be in play"); + } // a live event arriving *during* the probe would also relax the window, so record anything that // shows up while we are here; the witness sees it too, and the two together tell them apart @@ -249,6 +267,11 @@ private async Task ProbeAsync(ProvisionedDatabase database, int minutes, notes = live.Count == 0 ? string.Empty : $"also received live: {string.Join(", ", live)}"; } + if (received.Count != 0) + { + notes = string.IsNullOrEmpty(notes) ? received[0] : $"{received[0]}; {notes}"; + } + Note($" probe {minutes}m/{label}: relaxed={relaxed} type={type} {notes}"); return new Probe(minutes, label, relaxed, type, notes, Conclusive: true); } @@ -270,6 +293,51 @@ private void Note(string message) _progress?.WriteLine(message.Length == 0 ? message : $"{DateTime.UtcNow:HH:mm:ss} {message}"); } + /// Reads the notification type back out of the log line, which is the only place it is stated. + private static MaintenanceNotificationType TypeOf(string logLine) + { + foreach (var candidate in Enum.GetValues()) + { + if (candidate != MaintenanceNotificationType.None + && logLine.Contains(candidate.ToString(), StringComparison.Ordinal)) + { + return candidate; + } + } + + return MaintenanceNotificationType.None; + } + + /// Captures the client's own "maintenance notification" lines, attached before connecting. + private sealed class NotificationLog : ILoggerFactory, ILogger + { + private readonly List _received = []; + + public List Received + { + get { lock (_received) return [.. _received]; } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + if (message.Contains("Maintenance notification:", StringComparison.Ordinal)) + { + lock (_received) _received.Add(message); + } + } + + public void Dispose() { } + } + private static bool IsCompletion(MaintenanceNotificationType type) => type is MaintenanceNotificationType.Migrated or MaintenanceNotificationType.FailedOver; From 3572a7012f74069c60b539674fd96f9738237bad Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 17:51:02 +0100 Subject: [PATCH 18/36] Stop claiming parity with go-redis on the post-event tail The XML docs said the 20s default 'matches go-redis at the prescribed default'. It does not: go-redis restores normal timeouts on the end marker and node-redis decrements a counter clamped at zero, so neither keeps any tail after a completion (confirmed by their maintainers). Likely mis-attributed from go-redis's post-handoff relaxed duration, which is relaxation after a connection is replaced rather than after an event completes. States our own reasoning instead - the herd that re-engages the moment an event completes - names the divergence, and points at TimeSpan.Zero for the other clients' behaviour. --- .../Configuration/DefaultOptionsProvider.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index e71795334..fb5f88c66 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -392,9 +392,19 @@ protected virtual string GetDefaultClientName() => /// /// Gets how long timeouts stay relaxed after a disruption reports completion, or null to derive - /// it as twice the effective - /// (which matches go-redis at the prescribed default). + /// it as twice the effective . /// + /// + /// A completing notification says the server-side operation finished, not that the server is back to + /// normal latency - and the moment after one is precisely when every client that received the same + /// notification re-engages at once. That herd is the reason for a tail, and it scales with client + /// count, which is not something a client can observe. + /// + /// Note this is a deliberate divergence: go-redis and node-redis both treat a completion as + /// "stop being relaxed" and keep no tail at all (confirmed by their maintainers, 2026-09-02). Set this + /// to for that behaviour. + /// + /// [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] public virtual TimeSpan? MaintenancePostEventRelaxedDuration => null; From 51d22f042c0b1a512861945e834c3c8429febb21 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 17:53:02 +0100 Subject: [PATCH 19/36] Say that the connect-time rule is completions only The prose could be read as 'anything seen at connect time is disregarded'. A starter arriving then is not a replay - nothing retains those - so it still relaxes timeouts: it is a late-joining connection being told what remains of a disruption in progress. --- docs/ServerMaintenanceEvent.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 9e1b7aded..74c86e206 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -179,6 +179,8 @@ Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_ Because of that last point, a completion that arrives while the connection is still being established does **not** relax timeouts: it is history, not news. A completion that arrives on a live connection does, as the table above says. If you act on these events yourself, treat one that arrives at connection time as "this happened at some point", not "this is happening". +That applies to completions only. A *starter* arriving as you connect is not a replay - nothing retains those - so it still relaxes timeouts: it is the server telling a late-joining connection what is left of a disruption already in progress, which is when patience is most useful. + Note that a deliberate handoff appears as a `ConnectionFailed` event with `FailureType == ConnectionFailureType.MaintenanceHandoff`. That is expected during planned maintenance and does not indicate a fault; if you alert on `ConnectionFailed`, filter it out. ## Watching the events From acd830915a128ba418817d85352432e57a1f4228 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 18:07:04 +0100 Subject: [PATCH 20/36] Retention measured to three hours, not ninety minutes The ladder finished while this branch was being written: every probe from 1 to 180 minutes was handed the same completion, and the run ended because the rungs ran out rather than because anything expired. Updates the figure in the comments and docs that cite it as the reason for the catch-up rule. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalConnection.Maintenance.cs | 2 +- src/StackExchange.Redis/ServerEndPoint.Maintenance.cs | 2 +- tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs | 2 +- tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 74c86e206..d4f470ae9 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -175,7 +175,7 @@ Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_ * Only completions are replayed. Starters (`MIGRATING`, `FAILING_OVER`) are not, and neither is `MOVING` - so a replay can never demand that you move. * One item, most-recent-replaces; there is no queue. * It arrives within milliseconds of the opt-in being accepted, which is *during* connection establishment - so an event handler attached after `ConnectAsync` returns will usually not see it. -* **It can be very old.** The same `FAILED_OVER` was still being replayed to fresh connections 90 minutes after the failover, with no expiry observed, and a completion carries no time field - so nothing in the notification says how old it is. +* **It can be very old.** The same `FAILED_OVER` was still being replayed to fresh connections **three hours** after the failover - the longest anybody has measured, and it had not expired then - and a completion carries no time field, so nothing in the notification says how old it is. Because of that last point, a completion that arrives while the connection is still being established does **not** relax timeouts: it is history, not news. A completion that arrives on a live connection does, as the table above says. If you act on these events yourself, treat one that arrives at connection time as "this happened at some point", not "this is happening". diff --git a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs index 670e1382e..581baf83f 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs @@ -126,7 +126,7 @@ private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, P // A notification that arrives before the bridge reports established is the server's *catch-up* copy: // it retains the completion of a shard-scoped event and replays it to whoever opts in next, with no - // measured age limit (the same FAILED_OVER came back 90 minutes later). Distinguishing the two matters + // measured age limit (the same FAILED_OVER came back three hours later). Distinguishing the two matters // for the completions, which otherwise relax timeouts on a brand-new connection for an event that // finished long ago; the starters are unaffected, since nothing retains them. var isCatchUp = BridgeCouldBeNull?.IsConnected != true; diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 8fb899b71..eb441fed5 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -248,7 +248,7 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? // A completion delivered while we were still connecting is the server's catch-up channel, and // measurement says that channel has no age limit: the same FAILED_OVER was replayed to fresh - // connections 90 minutes after the failover, and completions carry no time field, so nothing in the + // connections three hours after the failover, and completions carry no time field, so nothing in the // frame distinguishes "just happened" from "happened this morning". Without this, every new // connection to a database that had ever failed over began life with the full post-event tail of // relaxed timeouts, and reported any timeout inside it as caused by maintenance that was long over. diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 9608efa36..f89a35c15 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -559,7 +559,7 @@ public async Task RetainedCompletionIsReplayedToANewConnection(MaintenanceNotifi Assert.Contains("(catch-up)", received); // ...and it did *not* relax anything. Measured on a live deployment: the same completion is - // replayed to fresh connections at least 90 minutes after the event, and completions carry no + // replayed to fresh connections three hours after the event, and completions carry no // time field, so relaxing here would mean every new connection to a database that had ever // failed over began life patient about timeouts, and attributing any of them to maintenance // that was long finished. diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs index e4a5fbd6e..c5ebdb1ae 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs @@ -115,7 +115,7 @@ public async Task ClosingNotificationLeavesThePostEventTail() public async Task ACatchUpCompletionOpensNoTail() { // Measured on a live deployment: a completion is retained and replayed to whoever opts in next, with - // no age limit found (the same FAILED_OVER came back at least 90 minutes later), and completions carry + // no age limit found (the same FAILED_OVER came back three hours later), and completions carry // no time field - so nothing in the frame says how old it is. Relaxing on one would mean every new // connection to a database that had ever failed over started life patient about timeouts, and // reporting any that expired as caused by maintenance long since finished. From 8a387806fcf576b3d34b7fa5fbb8c83381a17463 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 10:12:47 +0100 Subject: [PATCH 21/36] Attribute a fault to a window that has just closed, and warn on a missed handoff Two gaps from the requirement mapping, both small and both invisible without looking for them. **Attribution across a closed window.** Timeouts are raised by a once-a-second sweep rather than at the deadline, and a command that timed out had already been waiting for its whole timeout before that - so a window covering the command's entire life can have closed before the exception is built. Reading the *active* type then reported None for a timeout maintenance plainly caused, which is the opposite of what MaintenanceType exists for. A window that closed no longer ago than the command could have been waiting now still counts, floored at a second for the sweep's own imprecision. That bound is the tightest one that catches every genuine case, and it is deliberately a bound rather than a certainty: without per-message state - which the timeout sweeps rule out, since they rely on head-of-line ordering - a client cannot know whether *this* command overlapped the window, only whether it could have. The public docs say so. Implementation note: it mirrors the deadline into a second field rather than stamping a "closed at" moment, because expiry is lazy - there is no instant at which a window closes, and the deadline already is that instant. **The missed-deadline warning (R.2).** The contract asks for the replacement to be fully established - handshake complete, not merely socket-connected - before the announced window runs out, and to say so when it is not. Nothing said so. It matters because the two outcomes are indistinguishable from outside: commands succeed either way, since the relaxed window covers the gap, so a handoff that took three times its budget looked exactly like one that worked. Only paths that actually replaced connections are checked; where the decision was to do nothing, waiting for the server to close the socket is the plan, so reconnecting after the deadline is intended rather than a miss. Tests come in pairs, each half being the other's control: attribution survives a closed window but not a long-closed one, and the warning fires on a handoff that misses while staying silent on one that works. Provoking the miss took a correction worth recording. A named successor pointing at a dead address does not do it - the in-process transport routes by logical endpoint, so the replacement connects anyway. Slowing the handshake does, but the latency has to be set *before* the notification: setting it afterwards races the handoff, and since jitter on a short window can be almost nothing and an in-process reconnect takes about a millisecond, the replacement can be established before the latency lands. That is a correct no-warning outcome and a flaky test - measured, once, in a two-core whole-suite run. --- docs/ServerMaintenanceEvent.md | 10 +- src/StackExchange.Redis/ExceptionFactory.cs | 16 +++- src/StackExchange.Redis/LoggerExtensions.cs | 6 ++ .../Maintenance/MaintenanceFaultSurface.cs | 10 +- .../ServerEndPoint.Maintenance.cs | 94 +++++++++++++++++++ .../MaintenanceNotificationTests.cs | 83 +++++++++++++++- .../MaintenanceRelaxationTests.cs | 53 +++++++++++ 7 files changed, 265 insertions(+), 7 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index d4f470ae9..df22da9f5 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -223,7 +223,15 @@ The tail applies to a completion that arrives on a live connection. A completion The announced duration is clamped rather than honoured literally. Windows as short as two seconds have been observed in practice, which is not long enough to cover a client reconnecting, and a client that trusted the announced value would stop being patient exactly when it mattered. The tail exists for the same reason in reverse: after a handoff, servers and other clients are still settling. -If a command does time out during a window, the exception carries the reason: `RedisTimeoutException.MaintenanceType` (and the same property on `RedisConnectionException`) names the notification that was in effect, which distinguishes "the deployment was moving" from "this query is slow". +If a command does time out during a window, the exception carries the reason: `RedisTimeoutException.MaintenanceType` (and the same property on `RedisConnectionException`) names the notification that was in effect, which distinguishes "the deployment was moving" from "this query is slow". A window that closed very recently still counts, because timeouts are reported by a once-a-second sweep and the command had already been waiting for its whole timeout before that - so the window that caused a timeout is often over by the time you see the exception. + +A handoff that does not get a replacement connection fully established before the announced window runs out is reported as a warning: + +``` +10.0.0.1:6379: Maintenance handoff did not establish a replacement within the announced 15000ms (interactive: False, subscription: True) +``` + +Worth watching for, because it is otherwise invisible: commands succeed either way, since the relaxed window covers the gap, so a handoff that took three times its budget looks exactly like one that worked. ## Checking that it is working diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index c2977de0e..bfd92ff5e 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -319,9 +319,19 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; - // if the server had announced a disruption, say so on the fault: "timeout" and "timeout during an - // announced failover" call for very different reactions from whoever reads the log - var maintenanceType = server?.ActiveMaintenanceType ?? Maintenance.MaintenanceNotificationType.None; + // If the server had announced a disruption, say so on the fault: "timeout" and "timeout during an + // announced failover" call for very different reactions from whoever reads the log. + // + // Deliberately not the *active* type. A command that timed out was outstanding for its whole + // timeout before the heartbeat noticed, so a window covering its entire life can already have + // closed by the time this runs - which used to report None for a timeout maintenance plainly + // caused. The timeout that applied is the bound on how far back to look; take the async one when + // this message was awaited, since the two can be configured very differently. + var applicableTimeout = message?.ResultBoxIsAsync == true + ? multiplexer.AsyncTimeoutMilliseconds + : multiplexer.TimeoutMilliseconds; + var maintenanceType = server?.GetMaintenanceTypeForFault(applicableTimeout) + ?? Maintenance.MaintenanceNotificationType.None; Exception ex = logConnectionException && lastConnectionException is not null ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index ea21d0708..f598f5187 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -785,6 +785,12 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "{Server}: Maintenance notifications refused ({Reason})")] internal static partial void LogInformationMaintenanceNotificationsRefused(this ILogger logger, ServerEndPointLogValue server, string reason); + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 122, + Message = "{Server}: Maintenance handoff did not establish a replacement within the announced {WindowMilliseconds}ms (interactive: {Interactive}, subscription: {Subscription})")] + internal static partial void LogWarningMaintenanceHandoffMissedDeadline(this ILogger logger, ServerEndPointLogValue server, long windowMilliseconds, bool interactive, bool subscription); + [LoggerMessage( Level = LogLevel.Information, EventId = 121, diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs index 57959cdce..fbcae6149 100644 --- a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs +++ b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs @@ -1,4 +1,4 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.CodeAnalysis; using RESPite; using StackExchange.Redis.Maintenance; @@ -25,6 +25,14 @@ public sealed partial class RedisTimeoutException /// happened inside that window - including the tail after the disruption reported completion. It does not /// promise that the maintenance *caused* the timeout, only that the two coincided; that is still the most /// useful thing to know when reading a log after the fact. + /// + /// "Inside that window" is deliberately generous at the trailing edge. Timeouts are raised by a + /// once-a-second sweep rather than at the instant they expire, and a command that timed out had already + /// been waiting for its whole timeout before that - so a window covering the command's entire life can + /// have closed before anybody built this exception. A window that closed no longer ago than the command + /// could have been waiting therefore still counts, which catches every genuine case at the cost of + /// occasionally naming a window that a *different*, later command merely followed. + /// /// [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] public MaintenanceNotificationType MaintenanceType { get; internal init; } diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index eb441fed5..99cbffd10 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -267,12 +267,14 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? if (tail <= TimeSpan.Zero) { Volatile.Write(ref _relaxedDeadlineTicks, 0); + Volatile.Write(ref _relaxedEndedTicks, NudgeFromZero(Environment.TickCount)); Multiplexer.Trace($"{type}: relaxation ended", ToString()); return; } var deadline = NudgeFromZero(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); Volatile.Write(ref _relaxedDeadlineTicks, deadline); + Volatile.Write(ref _relaxedEndedTicks, deadline); Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } @@ -383,6 +385,8 @@ internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalC private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? currentAddress) { + var watch = ValueStopwatch.StartNew(); + var replaced = false; try { // Spread the fleet, but only by a fraction of the window - see MaintenanceHandoff.GetJitter for why @@ -411,6 +415,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? { case HandoffAction.Recycle: await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + replaced = true; break; case HandoffAction.RecycleAtHalfWindow: // The contract's rule for "no replacement named", applied where there is nothing better to @@ -418,6 +423,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); await DrainThenRecycleAsync(window - jitter - (half > TimeSpan.Zero ? half : TimeSpan.Zero), decision.Reason).ForAwait(); + replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: // Point the next connection at the named address and replace the connections. Previously @@ -426,12 +432,15 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + replaced = true; break; default: // Nothing to do is a legitimate outcome, not a failure: the server closes the socket, the // reconnect re-resolves, and the relaxed window covers the gap. break; } + + if (replaced) await WarnIfNotEstablishedInTimeAsync(watch, window).ForAwait(); } catch (Exception ex) { @@ -474,6 +483,39 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) ToString()); } + /// + /// Warns when a replacement connection is not fully established by the time the announced window ends. + /// + /// + /// The contract asks for the new connection to be *fully established* - handshake complete, not merely + /// socket-connected - before the deadline, and to say so when that is exceeded. It is the difference + /// between a handoff that worked and one the server finished for us by closing the socket, and from the + /// outside those look identical: commands succeed either way, because the relaxed window covers the gap. + /// + /// Only reached when connections were actually replaced. Where the decision was to do nothing, waiting for + /// the server to close the socket *is* the plan, so reconnecting after the deadline is the intended path + /// rather than a miss, and warning about it would be noise. + /// + /// + private async Task WarnIfNotEstablishedInTimeAsync(ValueStopwatch watch, TimeSpan window) + { + var deadline = window.TotalMilliseconds; + while (watch.ElapsedMilliseconds < deadline) + { + if (isDisposed) return; // nothing to report about an endpoint that has gone away + if (IsConnected && (IsSubscriberConnected || !SupportsSubscriptions)) return; // in time + await Task.Delay(50).ForAwait(); + } + + if (isDisposed || (IsConnected && (IsSubscriberConnected || !SupportsSubscriptions))) return; + + Multiplexer.Logger?.LogWarningMaintenanceHandoffMissedDeadline( + new(this), + (long)window.TotalMilliseconds, + IsConnected, + IsSubscriberConnected); + } + [ThreadStatic] private static Random? _random; @@ -496,12 +538,64 @@ private void ExtendRelaxedWindow(TimeSpan duration, string cause) if (current != 0 && unchecked(candidate - current) <= 0) return; if (Interlocked.CompareExchange(ref _relaxedDeadlineTicks, candidate, current) == current) { + Volatile.Write(ref _relaxedEndedTicks, candidate); Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); return; } } } + /// + /// When the most recent window was due to end, whether or not it has; zero if there has never been one. + /// + /// + /// A mirror of the deadline rather than a record of when it was cleared, which is what makes it cheap: + /// expiry is lazy (it happens on the next read), so there is no single moment at which to stamp "the + /// window closed", and the deadline already *is* that moment. + /// + private int _relaxedEndedTicks; + + /// How long after a window closes a fault may still be attributed to it, at minimum. + /// + /// The bridge heartbeat raises timeouts on roughly a one-second cadence rather than at the deadline, so a + /// timeout can be reported up to about a second after the moment it actually expired. + /// + private const int FaultAttributionFloorMilliseconds = 1000; + + /// + /// Which notification to blame for a fault, which is a different question from + /// . + /// + /// + /// A command that timed out was outstanding for its whole timeout before anybody noticed, and the + /// heartbeat that notices runs about once a second - so by the time an exception is built, a window that + /// covered the command's entire life may already have closed. Reading the *active* type then reports + /// for a timeout that maintenance plainly caused, which is + /// the opposite of what this property exists for. + /// + /// So a window that closed no longer ago than the command could have been waiting still counts. That is + /// the tightest bound that catches every genuine case, and it is deliberately a bound rather than a + /// certainty: without per-message state - which the timeout sweeps rule out, see + /// - a client cannot know whether *this* command overlapped + /// the window, only whether it could have. + /// + /// + /// The timeout that applied to the faulted command. + internal MaintenanceNotificationType GetMaintenanceTypeForFault(int timeoutMilliseconds) + { + var active = ActiveMaintenanceType; + if (active != MaintenanceNotificationType.None) return active; + + var ended = Volatile.Read(ref _relaxedEndedTicks); + if (ended == 0) return MaintenanceNotificationType.None; // never had a window at all + + var since = unchecked(Environment.TickCount - ended); + var grace = Math.Max(timeoutMilliseconds, FaultAttributionFloorMilliseconds); + return since >= 0 && since <= grace + ? (MaintenanceNotificationType)Volatile.Read(ref _relaxedType) + : MaintenanceNotificationType.None; + } + /// /// Zero is the "no window" sentinel, so a deadline that lands on it moves by a tick. /// diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index f89a35c15..056e2bdc4 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -24,17 +24,46 @@ public class MaintenanceNotificationTests(ITestOutputHelper log) /// The notifications are RESP3 push frames, so every test here forces RESP3 rather than running per /// protocol: under RESP2 there is nothing to receive, which is covered by the opt-in tests instead. /// - private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync(ITestOutputHelper log) + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( + ITestOutputHelper log, + ILoggerFactory? loggerFactory = null) { var server = new InProcessTestServer(log); var config = server.GetClientConfig(defaultOnly: true); config.Protocol = RedisProtocol.Resp3; config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; // must be live, or the test is vacuous + config.LoggerFactory = loggerFactory; var conn = await ConnectionMultiplexer.ConnectAsync(config); return (server, conn, new EventCollector(conn)); } + /// Captures the library's own log lines, so a test can assert on one it emits. + private sealed class CapturingLoggerFactory : ILoggerFactory, ILogger + { + private readonly List _lines = []; + + public IReadOnlyList Lines + { + get { lock (_lines) return [.. _lines]; } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_lines) _lines.Add(formatter(state, exception)); + } + + public void Dispose() { } + } + private sealed class EventCollector { private readonly ConcurrentQueue _events = new(); @@ -726,6 +755,51 @@ await Poll.UntilAsync( } } + [Fact] + public async Task AHandoffThatMissesTheWindowSaysSo() + { + // The contract asks for the replacement to be *fully established* before the announced window runs + // out, and to report it when that is exceeded. Worth having because the two outcomes are otherwise + // indistinguishable from outside: commands succeed either way, since the relaxed window covers the + // gap, so a handoff that quietly took three times its budget looks exactly like one that worked. + // + // Provoked by making the *handshake* slow rather than the socket unreachable: the in-process transport + // routes by logical endpoint, so a named successor still connects here (which is why the sibling test + // checks intent rather than redirection). A reply-delaying server means the replacement connects but + // cannot finish establishing inside the window, which is precisely the case the warning is for. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + // Slowed *before* the notification, deliberately. Setting the latency afterwards races the + // handoff: the jitter on a short window can be almost nothing and an in-process reconnect takes + // about a millisecond, so the replacement can be established before the latency lands - a correct + // no-warning outcome, and a test that fails on a loaded machine. Measured: it did, once, in a + // two-core whole-suite run. + server.SetLatency(TimeSpan.FromSeconds(5)); + + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + server.SendMoving(null, timeSeconds: 2, newEndpoint: successor, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + Assert.True( + await Poll.UntilAsync( + () => logs.Lines.Any(l => l.Contains("did not establish a replacement", StringComparison.Ordinal)), + timeoutMilliseconds: 20_000), + "a handoff that misses the announced window should be reported as a warning"); + + var warning = logs.Lines.First(l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + log.WriteLine(warning); + Assert.Contains("2000ms", warning); // the window it was given, so the log says what was missed + + // let the server answer normally again, so teardown is not fighting the latency + server.SetLatency(TimeSpan.Zero); + } + } + [Fact] public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() { @@ -737,7 +811,8 @@ public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() // no current address for the DNS branch to compare against; the deciding half of that branch is covered // by MaintenanceHandoffTests against a supplied resolver. What this proves is the acting half: drain, // drop, and come back. - var (server, conn, events) = await ConnectAsync(log); + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); using (server) await using (conn) { @@ -785,6 +860,10 @@ await Poll.UntilAsync( timeoutMilliseconds: 5_000), "the recycle should be reported as a MaintenanceHandoff rather than silently"); + // ...and the deadline warning stays silent, which is the other half of AHandoffThatMissesTheWindow- + // SaysSo: a warning that fires on the path that worked is noise rather than a diagnostic. + Assert.DoesNotContain(logs.Lines, l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + lock (failures) { log.WriteLine($"reported: {string.Join(", ", failures)}"); diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs index c5ebdb1ae..a98c0c058 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs @@ -111,6 +111,59 @@ public async Task ClosingNotificationLeavesThePostEventTail() } } + [Fact] + public async Task AFaultIsStillAttributedToAWindowThatHasJustClosed() + { + // Async timeouts are raised by the bridge heartbeat, not at the deadline, so a command that timed out + // was already outstanding for its whole timeout before anybody looked - and a window covering its + // entire life can have closed by the time the exception is built. Reading the *active* type then + // reports None for a timeout maintenance plainly caused, which is the opposite of the point. + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 1); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false), "the window should have closed"); + + // closed, so nothing is relaxed any more... + Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + + // ...but a command with a 30s timeout could easily have spanned it, so a fault is still its doing + Assert.Equal(MaintenanceNotificationType.Migrated, endpoint.GetMaintenanceTypeForFault(30_000)); + log.WriteLine($"attributed to {endpoint.GetMaintenanceTypeForFault(30_000)} after the window closed"); + } + } + + [Fact] + public async Task AFaultIsNotAttributedToALongClosedWindow() + { + // The other half: the lookback is bounded by how long the command could have been waiting, so a + // short-timeout command that fails well after a window closed is not blamed on it. Without a bound + // this would attribute every timeout for the rest of the process to the last maintenance event. + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 1); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false)); + + // past the one-second floor that covers the heartbeat's own imprecision + await Task.Delay(1500); + Assert.Equal(MaintenanceNotificationType.None, endpoint.GetMaintenanceTypeForFault(100)); + + // ...while a command that *could* have been outstanding that long still is attributed + Assert.Equal(MaintenanceNotificationType.Migrated, endpoint.GetMaintenanceTypeForFault(30_000)); + } + } + [Fact] public async Task ACatchUpCompletionOpensNoTail() { From c246f9f874adffc52267e8f1634a8698aa315711 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 12:47:53 +0100 Subject: [PATCH 22/36] Do not raise an event for a replayed completion If we are ignoring it, we should not report it either. A retained completion opens no window, because it is history rather than news, and raising it anyway hands a consumer a notification they cannot date: nothing in the frame says whether the failover was seconds or hours ago, so any action taken on it is a guess. It stays in the log, marked "(catch-up)", which is the right place for "here is what the server mentioned on the way in". This also narrows the catch-up test itself, which was too broad. "Arrived before the bridge reported established" is the only signal available, but on its own it also catches a *live* notification that merely happens to land mid-handshake - a late-joining connection being told about a disruption in progress. Restricting it to the kinds the server actually retains (MIGRATED, FAILED_OVER, per measurement) keeps those live: SMIGRATED is not retained at all, so one arriving here is news, and it still both relaxes and reports, and still drives the topology re-read. Two public XML doc blocks were stale beyond this change and are corrected: PushMaintenanceEvent still described the feature as observation-only, which stopped being true when the client began acting on notifications. --- docs/ServerMaintenanceEvent.md | 4 +- .../Maintenance/PushMaintenanceEvent.cs | 15 +++++-- .../PhysicalConnection.Maintenance.cs | 40 ++++++++++++++++--- .../MaintenanceNotificationTests.cs | 7 +++- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index df22da9f5..2e8d08ca7 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -177,9 +177,9 @@ Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_ * It arrives within milliseconds of the opt-in being accepted, which is *during* connection establishment - so an event handler attached after `ConnectAsync` returns will usually not see it. * **It can be very old.** The same `FAILED_OVER` was still being replayed to fresh connections **three hours** after the failover - the longest anybody has measured, and it had not expired then - and a completion carries no time field, so nothing in the notification says how old it is. -Because of that last point, a completion that arrives while the connection is still being established does **not** relax timeouts: it is history, not news. A completion that arrives on a live connection does, as the table above says. If you act on these events yourself, treat one that arrives at connection time as "this happened at some point", not "this is happening". +Because of that last point, a replayed completion is **not surfaced at all**: it does not relax timeouts, and `ServerMaintenanceEvent` is not raised for it. Nothing in the frame says whether the failover was seconds or hours ago, so any action taken on it would be a guess - and it is not ignored quietly, it is logged (see below), which is the right place for "here is what the server mentioned on the way in". -That applies to completions only. A *starter* arriving as you connect is not a replay - nothing retains those - so it still relaxes timeouts: it is the server telling a late-joining connection what is left of a disruption already in progress, which is when patience is most useful. +That applies to the retained kinds only - `MIGRATED` and `FAILED_OVER`. A *starter* arriving as you connect is not a replay, and neither is `SMIGRATED`; nothing retains those, so one arriving mid-handshake is the server telling a late-joining connection about a disruption in progress. Those are raised and do relax timeouts, which is when patience is most useful. Note that a deliberate handoff appears as a `ConnectionFailed` event with `FailureType == ConnectionFailureType.MaintenanceHandoff`. That is expected during planned maintenance and does not indicate a fault; if you alert on `ConnectionFailed`, filter it out. diff --git a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs index 550d1e1de..0870dc502 100644 --- a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs +++ b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs @@ -12,10 +12,17 @@ namespace StackExchange.Redis.Maintenance; /// notification: the payloads are near-identical, and it keeps the handling in one place. /// /// -/// Observation only at present: receiving one of these raises -/// and does nothing else. Acting on them - relaxing -/// timeouts, then moving off a doomed endpoint - is deliberately separate work, so a consumer can watch what -/// its servers are announcing before any behaviour depends on it. +/// The client acts on these itself - relaxing timeouts for the duration, learning a new topology, and moving +/// off an endpoint that says it is going away - so an application that only watches is watching work that has +/// already happened. See the ServerMaintenanceEvent documentation for what each notification does. +/// +/// One notification is raised once, however many nodes announced it: every node broadcasts a given event, so +/// the copies are collapsed on their sequence number and names whichever node arrived +/// first. And one case is deliberately *not* raised: a completion that the server retained and replayed to a +/// connection opting in later. That is history rather than news - it carries no time, so its age is +/// unknowable - and it is recorded in the log instead of being handed to a consumer who could only guess at +/// what to do with it. +/// /// [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] public sealed class PushMaintenanceEvent : ServerMaintenanceEvent diff --git a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs index 581baf83f..a993b2a69 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs @@ -124,12 +124,16 @@ private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, P Trace($"maintenance notification: {raw}"); OnDetailLog($"maintenance notification: {raw}"); - // A notification that arrives before the bridge reports established is the server's *catch-up* copy: - // it retains the completion of a shard-scoped event and replays it to whoever opts in next, with no - // measured age limit (the same FAILED_OVER came back three hours later). Distinguishing the two matters - // for the completions, which otherwise relax timeouts on a brand-new connection for an event that - // finished long ago; the starters are unaffected, since nothing retains them. - var isCatchUp = BridgeCouldBeNull?.IsConnected != true; + // A *retained* notification arriving before the bridge reports established is the server's catch-up + // copy: Enterprise keeps the most recent shard-scoped completion and replays it to whoever opts in + // next, with no measured age limit (the same FAILED_OVER came back three hours later). + // + // Both halves of the test are load-bearing. "Before established" is the only signal available, since a + // completion carries no time; and restricting it to the retained kinds is what keeps a *live* + // notification that merely happens to land mid-handshake from being mistaken for history - a + // late-joining connection can legitimately be told about a disruption in progress, and `SMIGRATED` is + // not retained at all, so one arriving here is news. + var isCatchUp = IsRetained(type) && BridgeCouldBeNull?.IsConnected != true; // relax before reporting: the event handler is consumer code, and the window should already be open // by the time anyone sees the notification that opened it @@ -174,6 +178,17 @@ private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, P } } + // A catch-up copy is not reported at all. We ignore it internally - it opens no window, because it is + // history rather than news - and raising it anyway would hand a consumer a notification it cannot + // date: nothing in the frame says whether the failover was seconds or hours ago, so any action taken + // on it is a guess. It stays visible in the log (with "(catch-up)" against it), which is the right + // place for "here is what the server mentioned on the way in". + if (isCatchUp) + { + Trace($"{kind} seq {sequenceId} is a retained copy of a finished event; not raising it"); + return OutOfBandResult.Handled; + } + // Per-server work above, one event below: relaxation is per-connection and every connection is told, // but a consumer wants one callback per logical event rather than one per proxy that mentioned it if (muxer.TryClaimMaintenanceEvent(type, sequenceId)) @@ -287,6 +302,19 @@ private static bool IsWindowClosing(MaintenanceNotificationType type) => type is or MaintenanceNotificationType.FailedOver or MaintenanceNotificationType.SlotMigrated; + /// + /// Whether the server retains this notification and replays it to connections that opt in later. + /// + /// + /// Measured on Redis Enterprise rather than specified: the most recent shard-scoped *completion* is + /// retained, most-recent-replaces, and nothing else is - not the starters, not MOVING, and not the + /// slot-scoped cluster forms. That is what makes a replay unable to demand action, and it is why only + /// these two kinds can arrive as history. + /// + private static bool IsRetained(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.FailedOver; + /// /// Whether the contract gives this notification a time element. /// diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 056e2bdc4..a399312ce 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -596,6 +596,11 @@ public async Task RetainedCompletionIsReplayedToANewConnection(MaintenanceNotifi Assert.False(endpoint.IsMaintenanceRelaxed, "a catch-up completion should not open the post-event tail"); Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); + // ...and it is not reported to consumers either, which is the same decision applied + // consistently: we ignore it, so we do not hand somebody a notification they cannot date and + // therefore cannot act on correctly. The log line above is where it stays visible. + await events.AssertNoneAsync(); + // ...and it must not have disturbed the handshake it arrived in the middle of, which is the // other half of what this test is for: a push frame interleaved with our own handshake // replies must be dispatched out-of-band rather than matched against one of them @@ -677,7 +682,7 @@ public async Task RetentionReplacesRatherThanAccumulates() log.WriteLine(received); Assert.Contains(nameof(MaintenanceNotificationType.FailedOver), received); Assert.Contains("seq=6", received); - GC.KeepAlive(events); + await events.AssertNoneAsync(); // received, but not reported } } From 7cc75fed65755c09a3d6bb8374d7496a028fea8e Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 14:34:38 +0100 Subject: [PATCH 23/36] Run the destructive scenarios, supervised The bucket that was never run: a shard dies, a node dies, a proxy dies. Held back because they damage the cluster and an unattended run would leave a broken environment behind; run at the end of a cluster's life, behind a second gate (SER_FI_DESTRUCTIVE) because E2E_SCENARIO_TESTS says "you may create databases", not "you may kill nodes". Measured against RS on 2026-09-03, one provisioned replicated database: - proxy_failure (bdb-scoped): one SocketClosed at +2.2s, restored at +10.0s, read succeeded immediately after. The only one the client saw at all. - shard_failure (bdb-scoped): zero drops. On a replicated database the proxy holds the connection while the replica takes over, so it is invisible. - node_failure: rejected a bdb_id outright - "Invalid parameter 'node_id': got None, expected valid node ID" - so it is node-scoped, which the schema does not say. Against node 2 it took 62s and was also invisible, because that is not the node serving us. The scope differing per action is recorded in the InlineData, since being told by an exception is currently the only documentation of it. Two of the three therefore pass without exercising the client, and the test says so in its own output rather than leaving somebody to infer coverage that is not there: a run with no drops proves the deployment absorbed the failure, not that our recovery works. Making node_failure bite needs the node that actually serves the database - resolve the endpoint's address and match it against the cluster's node list; SER_FI_NODE_TO_KILL is the manual stand-in. --- .../DestructiveScenarioTests.cs | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs new file mode 100644 index 000000000..562b419e0 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The scenarios that break things rather than move them: a shard dies, a node dies, a proxy dies. +/// +/// +/// Held back from every unattended run until now, deliberately - these damage the cluster, and leaving a +/// broken environment behind costs more than the coverage is worth when nobody is watching. Run them at the +/// end of a cluster's life, supervised, which is what this is. +/// +/// The client has nothing feature-specific to do here: a shard or node failing is not announced, so there is no +/// notification to act on and no handoff to perform. That is exactly why they are worth running. Everything +/// this feature adds - relaxed windows, handoffs, endpoint retirement - sits on top of ordinary reconnect and +/// topology handling, so a *silent* failure is the control: if recovery from an unannounced death regressed, +/// the announced paths are resting on sand. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "destructive")] +public class DestructiveScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string EnableVariable = "SER_FI_DESTRUCTIVE"; + + /// + /// Opt-in beyond the tier's own gate, because these cannot be undone. + /// + /// + /// E2E_SCENARIO_TESTS says "you may create and delete databases"; it does not say "you may kill + /// nodes". A cluster that has to be re-provisioned is 10-15 minutes of somebody's afternoon, so the second + /// gate is the difference between a deliberate session and an expensive surprise. + /// + /// + /// Which node node_failure kills; overridable, because which node matters and only the operator knows. + /// + /// + /// Not node 1: that is where the cluster's own management sits on a default install, so killing it takes + /// the fault injector's access with it and the test cannot observe its own outcome. + /// + private static int NodeToKill => + int.TryParse(Environment.GetEnvironmentVariable("SER_FI_NODE_TO_KILL"), out var node) && node > 0 ? node : 2; + + private static bool Enabled => + string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase); + + [Theory] + [InlineData("shard_failure", "bdb_id")] + [InlineData("proxy_failure", "bdb_id")] + [InlineData("node_failure", "node_id")] // measured: this one is scoped to a *node*, and rejects bdb_id + public async Task AnUnannouncedFailureIsSurvived(string action, string scope) + { + if (!Enabled) Assert.Skip($"set {EnableVariable}=true to run the destructive scenarios; they damage the cluster"); + + // Provisioned rather than a template database: a shard dying wants replication behind it, and the + // environment's own databases are created without it (and this cluster has none at all). + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + log.WriteLine($"{action} against {database}"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + var key = $"fi-{action}"; + await db.StringSetAsync(key, "before"); + + var drops = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (drops) drops.Add(e.FailureType); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId}"); + } + }; + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + action, + new Dictionary + { + // The scope differs by action and the schema does not say so: shard_failure and + // proxy_failure take the database, node_failure takes a node - discovered by being told + // "Invalid parameter 'node_id': got None, expected valid node ID". + [scope] = scope == "bdb_id" ? database.BdbId.ToString() : NodeToKill.ToString(), + }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + // The parameters for this family are untyped in the injector's schema, so a rejection is a harness + // finding rather than a client one - and recording the message is the point, since it is the only + // documentation of what these actions want. + Assert.Skip($"the injector would not run '{action}' with {scope}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports {action} finished"); + + // Recovery is polled rather than timed: what matters is that the client gets there on its own, and how + // long a real cluster takes to bring a shard back is not ours to assert. + var recovered = await Poll.UntilAsync( + () => + { + try + { + return db.StringGet(key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 120_000, + pollMilliseconds: 1000); + + lock (drops) + { + log.WriteLine( + $" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered={recovered} after {drops.Count} drop(s): " + + (drops.Count == 0 ? "(none)" : string.Join(", ", drops.Distinct()))); + + // Read the drop count before reading anything into a pass. Measured 2026-09-03: only + // proxy_failure was visible to the client at all (one SocketClosed, restored ~8s later); + // shard_failure on a replicated database and node_failure against a node we were not connected + // through both produced *zero* drops. A run with no drops has proved that the deployment absorbed + // the failure, which is worth knowing - but it has not exercised our recovery path, so do not + // count it as coverage of one. Making node_failure bite needs the node that actually serves this + // database, which means resolving the endpoint's address and matching it against the cluster's + // node list; SER_FI_NODE_TO_KILL is the manual version of that. + if (drops.Count == 0) + { + log.WriteLine(" note: the client never lost a connection, so this run tested the deployment rather than the client"); + } + } + + Assert.True(recovered, $"the client should recover on its own from {action} without being told"); + + // and the data survived, which is the deployment's promise rather than ours - stated because a + // "recovery" that silently lost the key would otherwise pass + Assert.Equal("before", await db.StringGetAsync(key)); + } +} From b1aea16761dded89ea536404cb264a622145f50f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 14:43:57 +0100 Subject: [PATCH 24/36] Document the destructive gate in the tier README The tests landed without their entry in the README that documents every other gate, which is where somebody looks before running the tier. Includes the two findings that change how a green run should be read: the parameter scope differs per action, and two of the three were absorbed by the deployment without the client noticing. --- .../README.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index 7f3c33f28..a381b1acc 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -44,6 +44,30 @@ Two details that are not incidental: lease easily, and counting a dead environment as "no replay" would report an expiry at whatever minute the cluster went away. +## The destructive scenarios, behind a second gate + +`DestructiveScenarioTests` breaks things rather than moving them - a shard, a node, or a proxy is killed - so +it needs its own opt-in on top of the tier's: + +```bash +export SER_FI_DESTRUCTIVE=true # absent means skip +export SER_FI_NODE_TO_KILL=2 # optional; which node node_failure kills (default 2, never 1) +``` + +`E2E_SCENARIO_TESTS` says "you may create and delete databases"; it does not say "you may kill nodes", and a +cluster that has to be re-provisioned is 10-15 minutes of somebody's afternoon. Run these at the end of a +cluster's life, watching. + +Two things measured on 2026-09-03 that change how you read a green run: + +- **The scope differs per action**, and the injector's schema does not say so: `shard_failure` and + `proxy_failure` take `bdb_id`, `node_failure` takes `node_id` and rejects a `bdb_id` outright. +- **Only `proxy_failure` was visible to the client** (one `SocketClosed`, restored ~8s later). `shard_failure` + on a replicated database and `node_failure` against a node we were not connected through both produced zero + drops - the deployment absorbed them. That is worth knowing and is *not* coverage of our recovery path, so + the test says so in its output when it sees no drops. Node 1 is avoided by default because cluster + management usually lives there, and killing it takes the fault injector's own access with it. + ## Three states, deliberately distinct | state | behaviour | From 366c9fe703a5549a4a65bbcee571acf99ad57c26 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 15:43:29 +0100 Subject: [PATCH 25/36] Prove a refusing node is retired, and correct why it was in doubt The field ticket suggested pruning could be starved indefinitely: the customer's exception was on the Subscription bridge with last: SSUBSCRIBE, retirement requires the endpoint to be idle, idleness counts caller work, and a sharded subscribe backlogged on a bridge that can never be written would be caller work - for longer while a maintenance window is open, since relaxation raises the timeout the backlog sweep purges against. Measured, and it does not happen. Server selection will not pick a disconnected node, so within a heartbeat the caller's subscribe is re-aimed at a reachable sibling; what accumulates on the refusing node is only our own probe traffic, which is exactly what HasCallerWork() was narrowed to exclude. Idleness stays true and the retirement proceeds with the window still open. Both halves are asserted, because the second is what makes the first safe: the refusing node must be accumulating something (or the distinction under test is vacuous) and none of it may be a caller's. Two things this cost, worth recording: - Removing a node from the fake does not model a node that has gone away. The tunnel only intercepts endpoints TryGetNode still resolves, but the already-established in-process pipe survives the removal, so the client never reconnects and never fails. Hence the black-hole tunnel: fall through to a real socket against a loopback port that was bound and released. - The first version of this test moved the slot before letting any pressure build, so the resubscribe went to the survivor and the test passed having exercised nothing. The ordering is the test. It also corrects the reading of the ticket: last: SSUBSCRIBE names the last command *written* on that bridge, not a queued caller subscribe. The veto it implied needs no reachable candidate for the slot at all, and at that point one endpoint's retirement is not the interesting question. --- .../RetirementUnderMaintenanceTests.cs | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs diff --git a/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs b/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs new file mode 100644 index 000000000..04389bce9 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs @@ -0,0 +1,200 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Configuration; +using StackExchange.Redis.Server; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Retiring a node that refuses every connection while a sharded subscription was pinned to it, with a +/// maintenance window open - the shape the field ticket suggested could starve pruning indefinitely. +/// +/// +/// The worry, from the ticket: the customer's exception was on the Subscription bridge with +/// last: SSUBSCRIBE. Retirement requires the endpoint to be idle, idleness counts caller work, +/// and a sharded subscribe backlogged on a bridge that can never write it would be caller work - so the +/// topology could say the node is gone, pruning could want to retire it, and IsIdle() could veto it, +/// for longer while a maintenance window is open (relaxation raises the timeout that +/// CheckBacklogForTimeouts purges against). +/// +/// Measured, and the answer is reassuring: it does not happen, for a reason worth pinning down. Server +/// selection will not pick a disconnected node, so within a heartbeat the caller's subscribe is re-aimed at a +/// reachable sibling. What piles up on the refusing node is only our own traffic - autoconfigure +/// probes, keep-alives - which is exactly what HasCallerWork() was narrowed to exclude, so idleness is +/// true and the retirement proceeds. Both halves are asserted below, because the second is what makes the +/// first safe. +/// +/// +/// This also corrects the reading of the ticket: last: SSUBSCRIBE names the last command written +/// on that bridge, not a queued caller subscribe. The veto it implied needs a stronger precondition - no +/// reachable candidate for the slot at all - and at that point one endpoint's retirement is not the +/// interesting question. +/// +/// +[Collection(NonParallelCollection.Name)] +public class RetirementUnderMaintenanceTests(ITestOutputHelper log) +{ + /// + /// Sends chosen endpoints to a real socket instead of to the in-process server, so they are refused. + /// + /// + /// Removing a node from the fake is not enough to model a node that has gone away: the tunnel only + /// intercepts endpoints TryGetNode still resolves, but an already-established in-process pipe + /// survives the removal, so the client never reconnects and so never fails. Falling through to a real + /// socket against a loopback port that was bound and released gives connection-refused on every attempt, + /// which is what the field failure actually did. + /// + private sealed class BlackHoleTunnel(Tunnel inner) : Tunnel + { + private readonly HashSet _blackHoled = []; + + public void BlackHole(EndPoint endpoint) + { + lock (_blackHoled) _blackHoled.Add(endpoint); + } + + private bool IsBlackHoled(EndPoint endpoint) + { + lock (_blackHoled) return _blackHoled.Contains(endpoint); + } + + public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) + => IsBlackHoled(endpoint) + ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) + : inner.GetSocketConnectEndpointAsync(endpoint, cancellationToken); + + public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) + => IsBlackHoled(endpoint) + ? base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) + : inner.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); + } + + /// A loopback port that has been bound and released, so connecting to it is refused rather than dropped. + private static IPEndPoint GetRefusingEndPoint() + { + using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return (IPEndPoint)probe.LocalEndPoint!; + } + + [Fact] + public async Task ARefusingNodeAccumulatesOnlyOurOwnTrafficAndIsRetired() + { + using var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + var doomed = GetRefusingEndPoint(); + server.AddEmptyNode(doomed); + + // the channel has to live on the doomed node, so pin it there by slot rather than by hope + var channel = RedisChannel.Sharded("retire-me"); + var asKey = (RedisKey)(byte[])channel!; + Assert.True(server.Migrate(asKey, doomed), "the fake should have moved the channel's slot"); + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + config.AbortOnConnectFail = false; + config.ReconnectRetryPolicy = new LinearRetry(500); + var tunnel = new BlackHoleTunnel(server.Tunnel); + config.Tunnel = tunnel; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var mux = (ConnectionMultiplexer)conn; + + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + var subscriber = conn.GetSubscriber(); + await subscriber.SubscribeAsync(channel, (_, _) => { }); + Assert.True( + await Poll.UntilAsync(() => Equals(subscriber.SubscribedEndpoint(channel), doomed), timeoutMilliseconds: 5000), + $"the subscription should be pinned to {doomed}, but went to {subscriber.SubscribedEndpoint(channel)}"); + + var endpoint = mux.GetServerEndPoint(doomed, ServerProvenance.ClusterTopology); + + // A long window, opened while the node still answers - which is the only way it can be opened, and is + // the ordering the ticket had: the disruption is announced, and *then* the node goes away. + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 60, shardIds: "[\"1\"]"); + Assert.True( + await Poll.UntilAsync(() => endpoint.IsMaintenanceRelaxed, timeoutMilliseconds: 5000), + "the notification should have opened a relaxed window on the doomed server"); + + // ...and now it goes away for good, refusing every connection. No notification of any of that, which + // is the point. + tunnel.BlackHole(doomed); + conn.GetServer(doomed).SimulateConnectionFailure(SimulatedFailureType.All); + + // Then *let the pressure build*, before the topology is allowed to catch up. This ordering is the + // whole test: while our slot map still says the channel lives on the doomed node, the resubscribe + // machinery keeps aiming SSUBSCRIBE at a bridge that can never write it, and those pile into the + // backlog as caller work. Move the slot first - as the first version of this test did - and the + // resubscribe goes to the survivor instead, nothing accumulates, and the test passes having exercised + // nothing. The ticket's client was in exactly this state: dead endpoint, stale map, pending SSUBSCRIBE. + // Let it sit *before* the topology is allowed to catch up. This ordering is the whole test: while our + // slot map still says the channel lives on the doomed node, anything aimed at that slot is aimed at a + // bridge that can never write it. Move the slot first and the question never arises. + var survivor = mux.GetServerEndPoint(server.DefaultEndPoint, ServerProvenance.ClusterTopology); + await Task.Delay(3000); + log.WriteLine( + $"under pressure - doomed: callerWork={endpoint.HasCallerWork()} outstanding={endpoint.GetOutstandingCount()} " + + $"relaxed={endpoint.IsMaintenanceRelaxed} idle={endpoint.IsIdle()}; " + + $"subscribed to {subscriber.SubscribedEndpoint(channel)?.ToString() ?? "(nowhere yet)"}"); + + // The refusing node *is* accumulating work - if it were not, the rest of this proves nothing, because + // the distinction being tested would be vacuous... + Assert.True( + endpoint.GetOutstandingCount() > 0, + "the refusing node should be accumulating our own probe traffic; with nothing outstanding this test " + + "is not exercising the distinction that lets retirement proceed"); + + // ...and none of it is a caller's, which is what keeps idleness true. Selection will not pick a + // disconnected node, so the caller's subscribe was re-aimed at the reachable sibling within a + // heartbeat rather than queueing here for the relaxed timeout to eventually purge. + Assert.False( + endpoint.HasCallerWork(), + "a caller's work should not be queued on a node that cannot be written to while a reachable " + + "candidate for the slot exists"); + Assert.NotEqual(doomed, subscriber.SubscribedEndpoint(channel)); + GC.KeepAlive(survivor); + + // only now does the cluster admit it has gone + Assert.True(server.Migrate(asKey, server.DefaultEndPoint)); + Assert.True(server.RemoveNode(doomed), "the node should have been removed from the fake"); + + // Generations are driven rather than waited for, as in the quiet case: what is under test is the + // retirement, not the refresh trigger. Bounded so a regression fails rather than hangs. + for (int i = 0; i < 40 && conn.GetEndPoints().Contains(doomed); i++) + { + await mux.ReconfigureAsync(first: false, reconfigureAll: true, log: null, blame: null, cause: $"test-generation-{i}"); + if (i % 8 == 0) + { + log.WriteLine( + $"generation {i}: idle={endpoint.IsIdle()} callerWork={endpoint.HasCallerWork()} " + + $"outstanding={endpoint.GetOutstandingCount()} relaxed={endpoint.IsMaintenanceRelaxed} " + + $"subs={endpoint.GetCounters().Subscription.Subscriptions}"); + } + + await Task.Delay(100); + } + + log.WriteLine($"endpoints: {string.Join(", ", conn.GetEndPoints().Select(x => x.ToString()))}"); + log.WriteLine( + $"final: idle={endpoint.IsIdle()} callerWork={endpoint.HasCallerWork()} " + + $"outstanding={endpoint.GetOutstandingCount()} relaxed={endpoint.IsMaintenanceRelaxed}"); + + Assert.DoesNotContain(doomed, conn.GetEndPoints()); + + // and the caller's subscription is not collateral damage: it belongs to the surviving node now + Assert.True( + await Poll.UntilAsync(() => Equals(subscriber.SubscribedEndpoint(channel), server.DefaultEndPoint), timeoutMilliseconds: 10_000), + $"the subscription should have moved to the surviving node, but is on {subscriber.SubscribedEndpoint(channel)}"); + } +} From 13bb7d348b5c4e4596a51a75ec374308ca9fe4ca Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 16:07:25 +0100 Subject: [PATCH 26/36] Put the fault-injector tier in the solution It was already built by CI - Build.csproj globs tests/**/*.csproj - but absent from the IDE solution, so it was easy to edit without noticing and easy to forget. Nothing in CI runs tests by solution or traversal (the one test step names StackExchange.Redis.Tests explicitly), and even when the tier does run, every test skips without SER_FI_CONFIG_DIR and E2E_SCENARIO_TESTS - so this changes what a developer sees, not what CI does. --- StackExchange.Redis.slnx | 1 + 1 file changed, 1 insertion(+) diff --git a/StackExchange.Redis.slnx b/StackExchange.Redis.slnx index 8b5b222aa..f3a6ac32f 100644 --- a/StackExchange.Redis.slnx +++ b/StackExchange.Redis.slnx @@ -40,6 +40,7 @@ + From 5d603aef4dcbe634b79032f4de49a7f591e4722f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 2 Sep 2026 13:58:12 +0100 Subject: [PATCH 27/36] Re-read the topology when an endpoint only ever refuses Every existing path that re-reads the topology needs somebody else to notice first: a maintenance notification, a MOVED from a reachable node, or a peer's configuration broadcast. reconfigureNextFailure is only set once a connection has been *established*, so an endpoint that has never connected loops on the heartbeat's reconnect path with nobody to tell it otherwise. Measured in the field: a client dialled three removed Redis Cloud nodes for ~37 hours. The gating is backwards for that case - an endpoint we have never established is more suspect than one we established and lost - but the flag does guard something real, a dead endpoint times a retry loop times every client in a fleet. So this replaces the gate rather than removing it: three consecutive connect failures provoke the existing jittered, coalesced refresh, no more often than max(ConfigCheckSeconds, 5) seconds, skipping endpoints that are disposed or already retiring. It repeats rather than firing once, because the server-side topology may not have caught up on the first attempt. Note configCheckSeconds was never a rebuttal to this: it drives an INFO replication on an established interactive bridge, not a topology read. Tests use a tunnel that black-holes one advertised node onto a loopback port that was bound and released, so it is refused on every attempt while remaining a member of the topology the fake advertises - removing the node from the fake instead proves nothing, since the already-established in-process pipe survives and the client never reconnects. With the hook commented out the first test fails with the CLUSTER count flat across 30s of refusals. --- src/StackExchange.Redis/LoggerExtensions.cs | 6 + src/StackExchange.Redis/PhysicalBridge.cs | 15 +- src/StackExchange.Redis/ServerEndPoint.cs | 55 ++++++ .../ConnectFailureRefreshTests.cs | 180 ++++++++++++++++++ 4 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index f598f5187..e6a94edda 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -802,4 +802,10 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) EventId = 119, Message = "{Server}: Maintenance handoff: {Outcome}")] internal static partial void LogInformationMaintenanceHandoff(this ILogger logger, ServerEndPointLogValue server, string outcome); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 120, + Message = "{Server}: Re-reading topology after {Failures} consecutive connect failures")] + internal static partial void LogInformationRefreshingAfterConnectFailures(this ILogger logger, ServerEndPointLogValue server, int failures); } diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index a739e7df8..4a3dbd0ae 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -507,9 +507,20 @@ internal void OnDisconnected(ConnectionFailureType failureType, PhysicalConnecti } ServerEndPoint.OnDisconnected(this); - if (!isDisposed && Interlocked.Increment(ref failConnectCount) == 1) + if (!isDisposed) { - TryConnect(null); // try to connect immediately + var consecutive = Interlocked.Increment(ref failConnectCount); + if (consecutive == 1) + { + TryConnect(null); // try to connect immediately + } + + // An endpoint we cannot even connect to is evidence that what we believe about the + // deployment may be wrong - and until now nothing acted on that. The reconfigure-on-failure + // path is gated on reconfigureNextFailure, which is only ever set once a connection has + // been *established*, so a node that has only ever refused could be retried forever + // without anybody re-reading the topology. Measured in the field: 37 hours. + ServerEndPoint.OnRepeatedConnectFailure(consecutive); } } else if (physical == null) diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 2517f531b..e7398108c 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -934,6 +934,61 @@ static async Task OnEstablishingAsyncAwaited(PhysicalConnection connection, Task return Task.CompletedTask; } + /// + /// How many consecutive failures to connect justify re-reading the topology. + /// + /// + /// Three: enough to rule out a single transient refusal, few enough that recovery is seconds. The + /// number matters less than that there *is* one - the failure this addresses lasted 37 hours. + /// + private const int ConnectFailuresBeforeRefresh = 3; + + private int _lastConnectFailureRefreshTicks; + + /// + /// Called when a connection attempt to this endpoint has failed, with the consecutive failure count. + /// + /// + /// The gap this closes: every existing path that re-reads the topology needs somebody *else* to notice + /// first - a notification, a MOVED from a reachable node, a peer's config broadcast. A client + /// with quiet healthy connections and one endpoint that only ever refuses has nobody to tell it, so it + /// dials the dead address indefinitely. That is not hypothetical: a customer's client did exactly that + /// for 37 hours across a Redis Cloud node replacement. + /// + /// Rate-limited to , deliberately reusing the + /// knob that already means "how often may we re-read configuration" rather than inventing one. The + /// limit is the part that makes this safe: the existing gate exists to stop a stampede - a dead + /// endpoint, times a retry loop, times every client in a fleet, each issuing CLUSTER NODES - and + /// removing the gate without replacing the restraint would trade a stuck client for a thundering herd. + /// + /// + /// It repeats rather than firing once, because one refresh is not guaranteed to help: the topology may + /// not have been updated server-side yet. A permanently dead endpoint therefore prompts a re-read at + /// most once per interval until something changes, which is what makes recovery eventual rather than + /// lucky. + /// + /// + internal void OnRepeatedConnectFailure(int consecutiveFailures) + { + if (consecutiveFailures < ConnectFailuresBeforeRefresh || isDisposed) return; + + // nothing to learn about an endpoint we have already decided to let go of + if ((unselectableReasons & UnselectableFlags.Retiring) != 0) return; + + var interval = Math.Max(Multiplexer.RawConfig.ConfigCheckSeconds, 5) * 1000; + var now = Environment.TickCount; + var last = Volatile.Read(ref _lastConnectFailureRefreshTicks); + if (last != 0 && unchecked(now - last) < interval) return; + + if (Interlocked.CompareExchange(ref _lastConnectFailureRefreshTicks, NudgeFromZeroTicks(now), last) != last) return; + + Multiplexer.Logger?.LogInformationRefreshingAfterConnectFailures(new(this), consecutiveFailures); + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, $"{consecutiveFailures} consecutive connect failures"); + } + + /// Zero means "never", so a tick count that lands on it moves by one. + private static int NudgeFromZeroTicks(int ticks) => ticks == 0 ? 1 : ticks; + internal void OnFullyEstablished(PhysicalConnection connection, string source) { try diff --git a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs new file mode 100644 index 000000000..546ed18af --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis.Configuration; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Re-reading the topology when an endpoint will not accept a connection at all. +/// +/// +/// The gap: every other path that re-reads the topology needs somebody *else* to notice first - a maintenance +/// notification, a MOVED from a reachable node, a peer's configuration broadcast. A client with quiet +/// healthy connections and one endpoint that only ever refuses has nobody to tell it, and +/// reconfigureNextFailure is set only once a connection has been *established*, so a node that never +/// established could be retried indefinitely. +/// +/// Not hypothetical: a customer's client dialled three endpoints that no longer existed for 37 hours across a +/// Redis Cloud node replacement, recovering only when something unrelated finally provoked a re-read. +/// +/// +[Collection(NonParallelCollection.Name)] +public class ConnectFailureRefreshTests(ITestOutputHelper log) +{ + /// Counts inbound CLUSTER commands, so a test can see a topology read happen. + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0 && string.Equals(request.GetString(0), "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + + return base.Execute(client, in request); + } + } + + /// + /// Hands every endpoint except one to the in-process server, and lets that one fall through to a real + /// socket against a port nothing is listening on. The point is that the node is a perfectly good member of + /// the topology the client was told about, and is refused on every attempt, without a single notification - + /// so nothing but the client's own persistence can notice. + /// + private sealed class BlackHoleTunnel(Tunnel inner, EndPoint blackHoled) : Tunnel + { + public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) + => blackHoled.Equals(endpoint) + ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) + : inner.GetSocketConnectEndpointAsync(endpoint, cancellationToken); + + public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) + => blackHoled.Equals(endpoint) + ? base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) + : inner.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); + } + + /// A loopback port that has been bound and released, so connecting to it is refused rather than dropped. + private static IPEndPoint GetRefusingEndPoint() + { + using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return (IPEndPoint)probe.LocalEndPoint!; + } + + private (CountingServer Server, ConfigurationOptions Config, EndPoint Doomed, CapturingLogger Logger) Arrange(int configCheckSeconds) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + var doomed = GetRefusingEndPoint(); + + // a real member of the topology - it holds a slot, and CLUSTER SLOTS advertises it - that simply + // cannot be reached; the client learns about it from the healthy node and then dials it forever + server.AddEmptyNode(doomed); + server.Migrate((RedisKey)"leaving", doomed); + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.AbortOnConnectFail = false; + config.ConfigCheckSeconds = configCheckSeconds; // the refresh rate limit reuses this + config.ConnectTimeout = 2000; + config.ReconnectRetryPolicy = new LinearRetry(500); // else the default backoff makes this a minutes-long test + config.Tunnel = new BlackHoleTunnel(server.Tunnel, doomed); + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + return (server, config, doomed, logger); + } + + [Fact] + public async Task AnEndpointThatOnlyEverRefusesProvokesATopologyRead() + { + var (server, config, doomed, logger) = Arrange(configCheckSeconds: 5); + using (server) + { + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + var before = server.ClusterCommands; + log.WriteLine($"cluster commands before: {before}"); + + var refreshed = await Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 30_000); + + // dumped before the assertion, so a failure arrives with the evidence rather than just a verdict + log.WriteLine($"cluster commands after: {server.ClusterCommands}"); + log.WriteLine(logger.All); + Assert.True(refreshed, "repeated connect failures should have provoked a topology read"); + Assert.NotEmpty(logger.Matching("consecutive connect failures")); + } + } + + [Fact] + public async Task TheReadIsRateLimitedRatherThanOncePerFailure() + { + // The restraint is the part that makes this safe to do at all. The gate it replaces exists to prevent a + // stampede - a dead endpoint, times a retry loop, times every client in a fleet, each issuing a + // topology read - so trading a stuck client for a thundering herd would be no improvement. + const int WindowSeconds = 12, ConfigCheckSeconds = 5; + var (server, config, doomed, logger) = Arrange(ConfigCheckSeconds); + using (server) + { + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000)); + + await Task.Delay(TimeSpan.FromSeconds(WindowSeconds), TestContext.Current.CancellationToken); + + var attempts = logger.Matching("Resurrecting").Count; + var refreshes = logger.Matching("consecutive connect failures").Count; + log.WriteLine($"{attempts} connect attempts, {refreshes} topology reads in {WindowSeconds}s"); + + // the ratio is the assertion: many failures, few reads. The bound is generous because the + // heartbeat that drives both is only ~1s accurate, but it is nowhere near one-read-per-failure. + Assert.True(attempts >= 5, $"expected the endpoint to be retried repeatedly, but saw {attempts} attempts"); + var permitted = (WindowSeconds / ConfigCheckSeconds) + 2; + Assert.True(refreshes <= permitted, $"expected at most {permitted} rate-limited reads, but saw {refreshes}"); + } + } + + private sealed class CapturingLogger : ILoggerFactory, ILogger + { + private readonly List _messages = []; + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_messages) _messages.Add(formatter(state, exception)); + } + + public List Matching(string fragment) + { + lock (_messages) return _messages.FindAll(x => x.Contains(fragment, StringComparison.Ordinal)); + } + + public string All + { + get { lock (_messages) return string.Join("\n", _messages); } + } + + public void Dispose() { } + } +} From f0de7f4784f6bd32a5ae7ae168327618359373c1 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 14:51:09 +0100 Subject: [PATCH 28/36] One black-hole tunnel, not one per test class The connect-failure trigger and the retirement test were written on separate branches a day apart and each grew its own copy of the same fixture, along with the same bound-then-released port helper. Now shared, with the reasoning in one place: why removing a node from the fake does not model a node going away, and why refused beats dropped (a dropped SYN exercises connect timeouts instead, which is a different failure mode and a slower test). --- .../ConnectFailureRefreshTests.cs | 35 ++--------- .../Helpers/BlackHoleTunnel.cs | 63 +++++++++++++++++++ .../RetirementUnderMaintenanceTests.cs | 45 +------------ 3 files changed, 69 insertions(+), 74 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs diff --git a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs index 546ed18af..102e83c17 100644 --- a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs +++ b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -48,37 +48,10 @@ public override TypedRedisValue Execute(RedisClient client, in RedisRequest requ } } - /// - /// Hands every endpoint except one to the in-process server, and lets that one fall through to a real - /// socket against a port nothing is listening on. The point is that the node is a perfectly good member of - /// the topology the client was told about, and is refused on every attempt, without a single notification - - /// so nothing but the client's own persistence can notice. - /// - private sealed class BlackHoleTunnel(Tunnel inner, EndPoint blackHoled) : Tunnel - { - public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) - => blackHoled.Equals(endpoint) - ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) - : inner.GetSocketConnectEndpointAsync(endpoint, cancellationToken); - - public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) - => blackHoled.Equals(endpoint) - ? base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) - : inner.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); - } - - /// A loopback port that has been bound and released, so connecting to it is refused rather than dropped. - private static IPEndPoint GetRefusingEndPoint() - { - using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); - return (IPEndPoint)probe.LocalEndPoint!; - } - private (CountingServer Server, ConfigurationOptions Config, EndPoint Doomed, CapturingLogger Logger) Arrange(int configCheckSeconds) { var server = new CountingServer(log) { ServerType = ServerType.Cluster }; - var doomed = GetRefusingEndPoint(); + var doomed = BlackHoleTunnel.GetRefusingEndPoint(); // a real member of the topology - it holds a slot, and CLUSTER SLOTS advertises it - that simply // cannot be reached; the client learns about it from the healthy node and then dials it forever @@ -91,7 +64,9 @@ private static IPEndPoint GetRefusingEndPoint() config.ConfigCheckSeconds = configCheckSeconds; // the refresh rate limit reuses this config.ConnectTimeout = 2000; config.ReconnectRetryPolicy = new LinearRetry(500); // else the default backoff makes this a minutes-long test - config.Tunnel = new BlackHoleTunnel(server.Tunnel, doomed); + var tunnel = new BlackHoleTunnel(server.Tunnel); + tunnel.BlackHole(doomed); // refused from the outset: this endpoint never accepts a connection at all + config.Tunnel = tunnel; var logger = new CapturingLogger(); config.LoggerFactory = logger; return (server, config, doomed, logger); diff --git a/tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs b/tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs new file mode 100644 index 000000000..fd065c802 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Configuration; + +namespace StackExchange.Redis.Tests; + +/// +/// Wraps another and sends chosen endpoints to a real socket instead, so they are +/// refused. +/// +/// +/// The fixture for "a node that has gone away", which is harder to model than it looks. Removing a node from +/// the in-process fake is not enough: InProcTunnel only intercepts endpoints TryGetNode still +/// resolves, but an already-established in-process pipe survives the removal, so the client never reconnects +/// and therefore never fails. Falling through to a real socket against a loopback port that was bound and +/// released gives connection-refused on every attempt, which is what a departed node actually does. +/// +/// Endpoints can be black-holed after connecting, which is what lets a test establish a connection, get the +/// client into the state it wants, and only then take the node away. +/// +/// +internal sealed class BlackHoleTunnel(Tunnel inner) : Tunnel +{ + private readonly HashSet _blackHoled = []; + + /// A loopback port that has been bound and released, so connecting to it is refused rather than dropped. + /// + /// Refused rather than timing out is the point: a dropped SYN would exercise connect *timeouts*, which is + /// a different failure mode with very different timing, and would make any test built on it slow. + /// + public static IPEndPoint GetRefusingEndPoint() + { + using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return (IPEndPoint)probe.LocalEndPoint!; + } + + /// Stops intercepting this endpoint, so connecting to it is refused from now on. + public void BlackHole(EndPoint endpoint) + { + lock (_blackHoled) _blackHoled.Add(endpoint); + } + + private bool IsBlackHoled(EndPoint endpoint) + { + lock (_blackHoled) return _blackHoled.Contains(endpoint); + } + + public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) + => IsBlackHoled(endpoint) + ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) + : inner.GetSocketConnectEndpointAsync(endpoint, cancellationToken); + + public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) + => IsBlackHoled(endpoint) + ? base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) + : inner.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); +} diff --git a/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs b/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs index 04389bce9..38cea9ea8 100644 --- a/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs +++ b/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs @@ -42,54 +42,11 @@ namespace StackExchange.Redis.Tests; [Collection(NonParallelCollection.Name)] public class RetirementUnderMaintenanceTests(ITestOutputHelper log) { - /// - /// Sends chosen endpoints to a real socket instead of to the in-process server, so they are refused. - /// - /// - /// Removing a node from the fake is not enough to model a node that has gone away: the tunnel only - /// intercepts endpoints TryGetNode still resolves, but an already-established in-process pipe - /// survives the removal, so the client never reconnects and so never fails. Falling through to a real - /// socket against a loopback port that was bound and released gives connection-refused on every attempt, - /// which is what the field failure actually did. - /// - private sealed class BlackHoleTunnel(Tunnel inner) : Tunnel - { - private readonly HashSet _blackHoled = []; - - public void BlackHole(EndPoint endpoint) - { - lock (_blackHoled) _blackHoled.Add(endpoint); - } - - private bool IsBlackHoled(EndPoint endpoint) - { - lock (_blackHoled) return _blackHoled.Contains(endpoint); - } - - public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) - => IsBlackHoled(endpoint) - ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) - : inner.GetSocketConnectEndpointAsync(endpoint, cancellationToken); - - public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) - => IsBlackHoled(endpoint) - ? base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) - : inner.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); - } - - /// A loopback port that has been bound and released, so connecting to it is refused rather than dropped. - private static IPEndPoint GetRefusingEndPoint() - { - using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); - return (IPEndPoint)probe.LocalEndPoint!; - } - [Fact] public async Task ARefusingNodeAccumulatesOnlyOurOwnTrafficAndIsRetired() { using var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; - var doomed = GetRefusingEndPoint(); + var doomed = BlackHoleTunnel.GetRefusingEndPoint(); server.AddEmptyNode(doomed); // the channel has to live on the doomed node, so pin it there by slot rather than by hope From 956ce534fc36e898aa870bba28dfccd7457a6b31 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 14:57:03 +0100 Subject: [PATCH 29/36] Document the connect-failure topology refresh User-visible behaviour with nothing in the docs describing it: three consecutive failed connects to an endpoint now provoke a topology re-read, rate-limited to configCheckSeconds. It could not be written while the code and the docs were on separate branches without describing something that was not there; now they are on one branch. Includes the point that configCheckSeconds is not itself a periodic topology refresh - it drives an INFO replication on an established connection - because that is exactly the misreading that made this gap look covered. --- docs/Configuration.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/Configuration.md b/docs/Configuration.md index 0c9afc388..0c2f4871b 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -265,6 +265,29 @@ Both options can be customized or disabled (set to `""`), via the `.Configuratio These settings are also used by the `IServer.MakeMaster()` method, which can set the tie-breaker in the database and broadcast the configuration change message. The configuration message can also be used separately to primary/replica changes simply to request all nodes to refresh their configurations, via the `ConnectionMultiplexer.PublishReconfigure` method. +## Refreshing the topology after repeated connect failures + +An endpoint that refuses every connection is evidence that what the client believes about the deployment may +be wrong, so after **three consecutive** failed connection attempts to the same endpoint, the client re-reads +the topology - the same refresh a `MOVED` or a configuration announcement would have caused, jittered and +coalesced in the same way. + +This closes a real gap rather than a theoretical one. Every other path that re-reads the topology needs +somebody *else* to notice first: a redirect from a reachable node, a peer's configuration announcement, or a +maintenance notification. The internal flag that drives a refresh-on-failure is only set once a connection has +been *established*, so an endpoint that has never connected - because it was replaced while the client was +running, or was already gone at startup - could be retried indefinitely with nobody to say otherwise. Measured +in the field: a client dialled three removed nodes for around 37 hours. + +The re-read is rate-limited to `configCheckSeconds` (default 60), deliberately reusing the knob that already +means "how often may we re-read configuration" rather than adding one. That restraint is what makes it safe: +a permanently dead endpoint, times a retry loop, times every client in a fleet would otherwise be a great +many topology reads, so a dead endpoint prompts at most one re-read per interval until something changes. + +Note that `configCheckSeconds` on its own is *not* a periodic topology refresh - it drives an +`INFO replication` on an established connection, which is a replication-role check. This is the path that +notices an endpoint nobody can reach. + ## ReconnectRetryPolicy StackExchange.Redis automatically tries to reconnect in the background when the connection is lost for any reason. It keeps retrying until the connection has been restored. It would use ReconnectRetryPolicy to decide how long it should wait between the retries. From fa6b032fd1d3f4c5617e53c7a3c5ec5a8c78bf2b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 15:20:41 +0100 Subject: [PATCH 30/36] Docs: lead with the server-native notifications, and name the feature The page opened with the Azure pub/sub support and put the push-frame mechanism at the bottom, which is backwards: one reports, the other acts, and the second is where this is going. Reversed, with the intro's bullets in the same order and the cross-links pointing the right way. Also states what the feature is called, because it has three names and a reader searching for the wrong one finds nothing: - smart client handoffs, the cross-client contract name (node-redis ships a suite under exactly that name) - hitless upgrades, the same thing named after its purpose, which is the wording lettuce and redis-py use, and which the shared test infrastructure treats as a synonym - maintenance notifications, the mechanism, which is what we call it, go-redis names its module maintnotifications, and Jedis calls maintenance events Both names now appear in docs/index.md and docs/Configuration.md so either one finds the guide. And Resp3.md still listed smart client handoffs as "not yet implemented in SE.Redis", which stopped being true with this feature; it now links to the guide and notes the RESP3-only constraint. --- docs/Configuration.md | 2 + docs/Resp3.md | 9 ++- docs/ServerMaintenanceEvent.md | 144 ++++++++++++++++++--------------- docs/index.md | 2 +- 4 files changed, 87 insertions(+), 70 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 0c2f4871b..80dc94cf6 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -338,6 +338,8 @@ before; they can additionally be named in a configuration string if they overrid ## Maintenance notifications +Server-native maintenance notifications - *smart client handoffs*, also called *hitless upgrades* - are configured with the keys below; [ServerMaintenanceEvent](ServerMaintenanceEvent) is the guide to what they do. + Redis Enterprise and Redis Cloud can warn a connected client *before* a disruptive event - a shard migration, a failover, or an endpoint being replaced - so the client can act ahead of it rather than discover it by way of a broken connection. This requires RESP3, and the client asks for it per connection: diff --git a/docs/Resp3.md b/docs/Resp3.md index 59d7128b5..9f851d61e 100644 --- a/docs/Resp3.md +++ b/docs/Resp3.md @@ -1,12 +1,13 @@ -# RESP3 and StackExchange.Redis +# RESP3 and StackExchange.Redis RESP2 and RESP3 are evolutions of the Redis protocol, with RESP3 existing from Redis server version 6 onwards (v7.2+ for Redis Enterprise). The main differences are: 1. RESP3 can carry out-of-band / "push" messages on a single connection, where-as RESP2 requires a separate connection for out-of-band (pub/sub) messages - this single connection can be of huge benefit in high-usage servers, as it halves the number of connections required -2. RESP3 supports *additional* out-of-band messages that cannot be expressed in RESP2, which allows advanced features such as "smart client handoffs" (a family of - server maintenance notifications) - - these features (not yet implemented in SE.Redis) allow for greater stability in complex deployments +2. RESP3 supports *additional* out-of-band messages that cannot be expressed in RESP2, which allows advanced features such as "smart client handoffs" (also called + "hitless upgrades"; a family of server maintenance notifications) + - these features allow for greater stability in complex deployments, and are implemented in SE.Redis: see + [ServerMaintenanceEvent](ServerMaintenanceEvent) - note they are RESP3-only, so a connection that ends up on RESP2 does not get them 3. RESP3 can (when appropriate) convey additional semantic meaning about returned payloads inside the same result structure - this is *mostly* relevant to client libraries that do not explicitly interpret the results before exposing to the user, so this does not directly impact SE.Redis itself, but it is relevant to consumers of SE.Redis that use Lua scripts or ad-hoc commands diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 2e8d08ca7..4f5f85d3c 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -1,83 +1,31 @@ # Introducing ServerMaintenanceEvents -StackExchange.Redis now automatically subscribes to notifications about upcoming maintenance from supported Redis providers. The ServerMaintenanceEvent on the ConnectionMultiplexer raises events in response to notifications about server maintenance, and application code can subscribe to the event to handle connection drops more gracefully during these maintenance operations. +StackExchange.Redis automatically subscribes to notifications about upcoming maintenance from supported Redis providers. The `ServerMaintenanceEvent` on the `ConnectionMultiplexer` raises events in response to them, and application code can subscribe to handle connection drops more gracefully during these operations. -There are two sources of these events, and they arrive by completely different routes: +There are two sources, and they arrive by completely different routes: -* **Azure Cache for Redis** publishes them on a pub/sub channel (`AzureRedisEvents`), and they surface as `AzureMaintenanceEvent`. This is the original support, and is described below. -* **Redis Enterprise and Redis Cloud** send them as RESP3 *push frames* on the connection that carries your commands, and they surface as `PushMaintenanceEvent`. This is newer, does more than report, and is covered in [its own section](#server-native-maintenance-notifications-redis-enterprise-and-redis-cloud). +* **Redis Enterprise and Redis Cloud** send them as RESP3 *push frames* on the connection that carries your commands, and they surface as `PushMaintenanceEvent`. The client does not merely report these - it acts on them - and this is the direction the feature is going, so it is covered [first, below](#server-native-maintenance-notifications-redis-enterprise-and-redis-cloud). +* **Azure Cache for Redis** publishes them on a pub/sub channel (`AzureRedisEvents`), and they surface as `AzureMaintenanceEvent`. This is the original support, and is described [further down](#azure-cache-for-redis-maintenance-events-pubsub). Both raise the same `ServerMaintenanceEvent` event, so a handler can watch for either. If you are a Redis vendor and want to integrate support for ServerMaintenanceEvents into StackExchange.Redis, we recommend opening an issue so we can discuss the details. -## Types of events - -Azure Cache for Redis currently sends the following notifications: -* `NodeMaintenanceScheduled`: Indicates that a maintenance event is scheduled. Can be 10-15 minutes in advance. -* `NodeMaintenanceStarting`: This event gets fired ~20s before maintenance begins -* `NodeMaintenanceStart`: This event gets fired when maintenance is imminent (<5s) -* `NodeMaintenanceFailoverComplete`: Indicates that a replica has been promoted to primary -* `NodeMaintenanceEnded`: Indicates that the node maintenance operation is over - -## Sample code - -The library will automatically subscribe to the pub/sub channel to receive notifications from the server, if one exists. For Azure Redis caches, this is the 'AzureRedisEvents' channel. To plug in your maintenance handling logic, you can pass in an event handler via the `ServerMaintenanceEvent` event on your `ConnectionMultiplexer`. For example: - -```csharp -multiplexer.ServerMaintenanceEvent += (object sender, ServerMaintenanceEvent e) => -{ - if (e is AzureMaintenanceEvent azureEvent && azureEvent.NotificationType == AzureNotificationType.NodeMaintenanceStart) - { - // Take whatever action is appropriate for your application to handle the maintenance operation gracefully. - // This might mean writing a log entry, redirecting traffic away from the impacted Redis server, or - // something entirely different. - } -}; -``` -You can see the schema for the `AzureMaintenanceEvent` class [here](https://github.com/StackExchange/StackExchange.Redis/blob/main/src/StackExchange.Redis/Maintenance/AzureMaintenanceEvent.cs). Note that the library automatically sets the `ReceivedTimeUtc` timestamp when the event is received, so if you see in your logs that `ReceivedTimeUtc` is after `StartTimeUtc`, this may indicate that your connections are under high load. - -## Walking through a sample maintenance event - -1. App is connected to Redis and everything is working fine. -2. Current Time: [16:21:39] -> `NodeMaintenanceScheduled` event is raised, with a `StartTimeUtc` of 16:35:57 (about 14 minutes from current time). - * Note: the start time for this event is an approximation, because we will start getting ready for the update proactively and the node may become unavailable up to 3 minutes sooner. We recommend listening for `NodeMaintenanceStarting` and `NodeMaintenanceStart` for the highest level of accuracy (these are only likely to differ by a few seconds at most). -3. Current Time: [16:34:26] -> `NodeMaintenanceStarting` message is received, and `StartTimeUtc` is 16:34:46, about 20 seconds from the current time. -4. Current Time: [16:34:46] -> `NodeMaintenanceStart` message is received, so we know the node maintenance is about to happen. We break the circuit and stop sending new operations to the Redis connection. (Note: the appropriate action for your application may be different.) StackExchange.Redis will automatically refresh its view of the overall server topology. -5. Current Time: [16:34:47] -> The connection is closed by the Redis server. -6. Current Time: [16:34:56] -> `NodeMaintenanceFailoverComplete` message is received. This tells us that the replica node has promoted itself to primary, so the other node can go offline for maintenance. -7. Current Time [16:34:56] -> The connection to the Redis server is restored. It is safe to send commands again to the connection and all commands will succeed. -8. Current Time [16:37:48] -> `NodeMaintenanceEnded` message is received, with a `StartTimeUtc` of 16:37:48. Nothing to do here if you are talking to the load balancer endpoint (port 6380 or 6379). For clustered servers, you can resume sending readonly workloads to the replica(s). - -## Azure Cache for Redis Maintenance Event details - -#### NodeMaintenanceScheduled event - -`NodeMaintenanceScheduled` events are raised for maintenance scheduled by Azure, up to 15 minutes in advance. This event will not get fired for user-initiated reboots. - -#### NodeMaintenanceStarting event - -`NodeMaintenanceStarting` events are raised ~20 seconds ahead of upcoming maintenance. This means that one of the primary or replica nodes will be going down for maintenance. - -It's important to understand that this does *not* mean downtime if you are using a Standard/Premier SKU cache. If the replica is targeted for maintenance, disruptions should be minimal. If the primary node is the one going down for maintenance, a failover will occur, which will close existing connections going through the load balancer port (6380/6379) or directly to the node (15000/15001). You may want to pause sending write commands until the replica node has assumed the primary role and the failover is complete. - -#### NodeMaintenanceStart event - -`NodeMaintenanceStart` events are raised when maintenance is imminent (within seconds). These messages do not include a `StartTimeUtc` because they are fired immediately before maintenance occurs. - -#### NodeMaintenanceFailoverComplete event +# Server-native maintenance notifications (Redis Enterprise and Redis Cloud) -`NodeMaintenanceFailoverComplete` events are raised when a replica has promoted itself to primary. These events do not include a `StartTimeUtc` because the action has already occurred. +> These APIs are experimental, behind diagnostic id `SER010`; see [SER010](exp/SER010.md). -#### NodeMaintenanceEnded event +Redis Enterprise and Redis Cloud can tell a client *directly* that a disruption is coming: a shard is migrating, a node is failing over, or the endpoint you are connected to is being replaced. These arrive as RESP3 push frames on the connection itself, and the client does not merely report them: it relaxes timeouts for the duration, re-reads the cluster topology when slots have moved, recovers sharded subscriptions that were stranded, and moves off an endpoint that is going away rather than waiting to be disconnected. -`NodeMaintenanceEnded` events are raised to indicate that the maintenance operation has completed and that the replica is once again available. You do *NOT* need to wait for this event to use the load balancer endpoint, as it is available throughout. However, we included this for logging purposes and for customers who use the replica endpoint in clusters for read workloads. +### What this feature is called -# Server-native maintenance notifications (Redis Enterprise and Redis Cloud) +One feature, several names, which matters mostly when you are searching: -> These APIs are experimental, behind diagnostic id `SER010`; see [SER010](exp/SER010.md). +* **Smart client handoffs** is the name used across Redis's client libraries for the cross-client contract this implements - for example node-redis has a `smart-client-handoffs` end-to-end suite. +* **Hitless upgrades** is the same thing named after its purpose: an upgrade or rollout that does not drop the caller's work. Lettuce and redis-py both use this wording for their coverage of it, and the Redis test infrastructure treats "hitless upgrade" and "smart client handoff" as synonyms. +* **Maintenance notifications** is the name of the mechanism, and what this library calls it - go-redis names its module `maintnotifications`, and Jedis calls them maintenance events. -Redis Enterprise and Redis Cloud can tell a client *directly* that a disruption is coming: a shard is migrating, a node is failing over, or the endpoint you are connected to is being replaced. Unlike the Azure events above, these arrive as RESP3 push frames on the connection itself, and the client does not merely report them: it relaxes timeouts for the duration, re-reads the cluster topology when slots have moved, recovers sharded subscriptions that were stranded, and moves off an endpoint that is going away rather than waiting to be disconnected. +Server-side you may also see it discussed as *maintenance mode*, *shard migration* and *endpoint rebinding*, which are the operations that emit the notifications rather than names for the feature. ## Do I need to configure anything? @@ -274,3 +222,69 @@ Alternatively set `MaintenanceNotifications = Enabled` in a test or staging envi ## Which deployments send these Redis Enterprise and Redis Cloud send them, subject to the feature being enabled on the cluster. Azure Managed Redis is configured to ask for them ahead of its own rollout, so the setting is harmless until their servers begin emitting. Redis Open Source, Valkey and other servers do not send them at all, and the setting is simply inert there: the opt-in is refused and the client carries on. + +# Azure Cache for Redis maintenance events (pub/sub) + +The original support, and unrelated to the push-frame mechanism above: Azure Cache for Redis publishes maintenance notifications on the `AzureRedisEvents` pub/sub channel, and the client reports them without changing its own behaviour. + +## Types of events + +Azure Cache for Redis currently sends the following notifications: +* `NodeMaintenanceScheduled`: Indicates that a maintenance event is scheduled. Can be 10-15 minutes in advance. +* `NodeMaintenanceStarting`: This event gets fired ~20s before maintenance begins +* `NodeMaintenanceStart`: This event gets fired when maintenance is imminent (<5s) +* `NodeMaintenanceFailoverComplete`: Indicates that a replica has been promoted to primary +* `NodeMaintenanceEnded`: Indicates that the node maintenance operation is over + +## Sample code + +The library will automatically subscribe to the pub/sub channel to receive notifications from the server, if one exists. For Azure Redis caches, this is the 'AzureRedisEvents' channel. To plug in your maintenance handling logic, you can pass in an event handler via the `ServerMaintenanceEvent` event on your `ConnectionMultiplexer`. For example: + +```csharp +multiplexer.ServerMaintenanceEvent += (object sender, ServerMaintenanceEvent e) => +{ + if (e is AzureMaintenanceEvent azureEvent && azureEvent.NotificationType == AzureNotificationType.NodeMaintenanceStart) + { + // Take whatever action is appropriate for your application to handle the maintenance operation gracefully. + // This might mean writing a log entry, redirecting traffic away from the impacted Redis server, or + // something entirely different. + } +}; +``` +You can see the schema for the `AzureMaintenanceEvent` class [here](https://github.com/StackExchange/StackExchange.Redis/blob/main/src/StackExchange.Redis/Maintenance/AzureMaintenanceEvent.cs). Note that the library automatically sets the `ReceivedTimeUtc` timestamp when the event is received, so if you see in your logs that `ReceivedTimeUtc` is after `StartTimeUtc`, this may indicate that your connections are under high load. + +## Walking through a sample maintenance event + +1. App is connected to Redis and everything is working fine. +2. Current Time: [16:21:39] -> `NodeMaintenanceScheduled` event is raised, with a `StartTimeUtc` of 16:35:57 (about 14 minutes from current time). + * Note: the start time for this event is an approximation, because we will start getting ready for the update proactively and the node may become unavailable up to 3 minutes sooner. We recommend listening for `NodeMaintenanceStarting` and `NodeMaintenanceStart` for the highest level of accuracy (these are only likely to differ by a few seconds at most). +3. Current Time: [16:34:26] -> `NodeMaintenanceStarting` message is received, and `StartTimeUtc` is 16:34:46, about 20 seconds from the current time. +4. Current Time: [16:34:46] -> `NodeMaintenanceStart` message is received, so we know the node maintenance is about to happen. We break the circuit and stop sending new operations to the Redis connection. (Note: the appropriate action for your application may be different.) StackExchange.Redis will automatically refresh its view of the overall server topology. +5. Current Time: [16:34:47] -> The connection is closed by the Redis server. +6. Current Time: [16:34:56] -> `NodeMaintenanceFailoverComplete` message is received. This tells us that the replica node has promoted itself to primary, so the other node can go offline for maintenance. +7. Current Time [16:34:56] -> The connection to the Redis server is restored. It is safe to send commands again to the connection and all commands will succeed. +8. Current Time [16:37:48] -> `NodeMaintenanceEnded` message is received, with a `StartTimeUtc` of 16:37:48. Nothing to do here if you are talking to the load balancer endpoint (port 6380 or 6379). For clustered servers, you can resume sending readonly workloads to the replica(s). + +## Azure Cache for Redis Maintenance Event details + +#### NodeMaintenanceScheduled event + +`NodeMaintenanceScheduled` events are raised for maintenance scheduled by Azure, up to 15 minutes in advance. This event will not get fired for user-initiated reboots. + +#### NodeMaintenanceStarting event + +`NodeMaintenanceStarting` events are raised ~20 seconds ahead of upcoming maintenance. This means that one of the primary or replica nodes will be going down for maintenance. + +It's important to understand that this does *not* mean downtime if you are using a Standard/Premier SKU cache. If the replica is targeted for maintenance, disruptions should be minimal. If the primary node is the one going down for maintenance, a failover will occur, which will close existing connections going through the load balancer port (6380/6379) or directly to the node (15000/15001). You may want to pause sending write commands until the replica node has assumed the primary role and the failover is complete. + +#### NodeMaintenanceStart event + +`NodeMaintenanceStart` events are raised when maintenance is imminent (within seconds). These messages do not include a `StartTimeUtc` because they are fired immediately before maintenance occurs. + +#### NodeMaintenanceFailoverComplete event + +`NodeMaintenanceFailoverComplete` events are raised when a replica has promoted itself to primary. These events do not include a `StartTimeUtc` because the action has already occurred. + +#### NodeMaintenanceEnded event + +`NodeMaintenanceEnded` events are raised to indicate that the maintenance operation has completed and that the replica is once again available. You do *NOT* need to wait for this event to use the load balancer endpoint, as it is available throughout. However, we included this for logging purposes and for customers who use the replica endpoint in clusters for read workloads. diff --git a/docs/index.md b/docs/index.md index 0665045ed..aacd8021f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,7 +45,7 @@ Documentation - [Pub/Sub Key Notifications](KeyspaceNotifications) - how to use keyspace and keyevent notifications - [Hot Keys](HotKeys) - how to use `HOTKEYS` profiling - [Using RESP3](Resp3) - information on using RESP3 -- [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance, including the server-native notifications sent by Redis Enterprise and Redis Cloud +- [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance, including the server-native notifications sent by Redis Enterprise and Redis Cloud (known elsewhere as *smart client handoffs* or *hitless upgrades*) - [Streams](Streams) - how to use the Stream data type - [Arrays](Arrays) - how to use Redis Arrays as sparse arrays of values - [Vector Sets](VectorSets) - how to use Vector Sets for similarity search with embeddings From eb8c4a013b54b6e311659c556e1849e478a2fa44 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 15:27:38 +0100 Subject: [PATCH 31/36] Soften the SequenceId wording "Observed behaviour, not a contract" led with a disclaimer and buried what the field is for. Both the guide and the XML docs now say what it does - it identifies the event rather than the delivery, so the same notification arriving from several proxies, or replayed after a reconnect, is recognisable as one event - and give the practical caveat as advice rather than as a warning: use it for correlation and de-duplication, not as an arithmetic sequence, because gaps are normal when a client only sees its own events. SER010's justification for the gate keeps its frankness, since explaining why the API is experimental is the point of that page, but drops the claim that no captured transcript exists to validate against. That was true when it was written and is not now. --- docs/ServerMaintenanceEvent.md | 2 +- docs/exp/SER010.md | 10 +++++----- .../Maintenance/PushMaintenanceEvent.cs | 17 +++++++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 4f5f85d3c..c253475fc 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -153,7 +153,7 @@ multiplexer.ServerMaintenanceEvent += (sender, e) => Two things are worth knowing before you build on the detail: * **`EndPoint` is whichever node told us first.** Every node broadcasts a given event, so the client collapses the copies and raises one event; it is not necessarily the node being maintained, and for the cluster notifications it is usually a bystander reporting somebody else's movements. -* **`SequenceId` is observed behaviour, not a contract.** No specification defines it. In practice it is monotonic per database and shared across notification types, which makes it useful for spotting a replay, but do not depend on it across deployments or versions. +* **`SequenceId` identifies the event, not the delivery.** Every node that broadcasts a given event carries the same value, so the same notification arriving from several proxies - or replayed after a reconnect - is recognisable as one event. Ids are allocated per database and shared across notification types, so a `SMIGRATING` at 16 is followed by its `SMIGRATED` at 17. Use it for correlation and de-duplication rather than as an arithmetic sequence: gaps are normal, because a client only sees the events relevant to it. `Time` is what the server announced, and may legitimately be zero or negative for a connection that arrived mid-window, meaning "this is happening now". diff --git a/docs/exp/SER010.md b/docs/exp/SER010.md index 2645b9393..a09ccd279 100644 --- a/docs/exp/SER010.md +++ b/docs/exp/SER010.md @@ -1,4 +1,4 @@ -Maintenance notifications ("smart client handoffs") are a server feature by which a server warns a +Maintenance notifications ("smart client handoffs") are a server feature by which a server warns a connected client in advance of a disruptive event - a shard migration, a failover, or the endpoint itself being replaced - so the client can act before the disruption rather than react to a broken connection afterwards. The client opts in per connection with `CLIENT MAINT_NOTIFICATIONS ON`, and the @@ -9,10 +9,10 @@ The feature is experimental here for three separate reasons: 1. **The server side is not universal.** Only Redis Enterprise and Redis Cloud emit these notifications today; OSS Redis, Valkey and Garnet do not recognize the opt-in at all. That is why `MaintenanceNotificationMode.Auto` exists, and why the default is `Disabled`. -2. **The wire contract is still moving.** Notification types have been proposed and withdrawn during - its development, the semantics of some fields (notably the sequence number) are not defined by any - specification, and no captured RESP transcript exists to validate against. Our parsing is - deliberately liberal, but the shapes may change. +2. **The wire contract is still evolving.** Notification types have been proposed and withdrawn during + its development, and the payloads are described in prose rather than pinned down by a formal + specification. Our parsing is validated against frames captured from live deployments and is + deliberately liberal about what it accepts, but the shapes may still change. 3. **What the client *does* in response is the substantial part**, and it is being built in stages - timeout relaxation, then endpoint handoff. Behaviour may therefore change materially between versions while the diagnostic is in place, even where the API does not. diff --git a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs index 0870dc502..67d2ac955 100644 --- a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs +++ b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs @@ -60,15 +60,16 @@ internal PushMaintenanceEvent( /// The sequence number the server attached to this notification. /// /// - /// No specification defines these, but observation does: on Enterprise 8.6.2 they are monotonic per - /// database, start at zero on a fresh one, are shared *across* notification types (a - /// at 16 followed by its - /// at 17), and carry the same value on every node - /// that broadcasts a given event - so they identify the event rather than the connection that delivered - /// it. That makes them genuinely useful for spotting a replay. + /// These identify the *event* rather than the connection that delivered it, which is what makes them + /// useful: every node that broadcasts a given event carries the same value, so a notification arriving + /// twice - once per proxy, or replayed on a reconnect - is recognisable as the same one. The client uses + /// them for exactly that, and so can you. /// - /// Still treat cross-deployment use as heuristic: this is one build of one product, and nothing obliges a - /// different implementation to behave the same way. + /// They are allocated per database, ascending, and shared across notification types, so a + /// at 16 is followed by its + /// at 17. Best used for correlating and + /// de-duplicating notifications rather than as an arithmetic sequence: gaps are normal, since a client + /// only sees the events relevant to it. /// /// public long SequenceId { get; } From a9d3f1c45b2209e01a5b3111463c2ea857f19622 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 15:47:30 +0100 Subject: [PATCH 32/36] Validate endpoint retirement against a real node removal The destructive scenario that exercises something this feature built. Pruning exists because the endpoint collection used to be add-only - a node that left the cluster was dialled forever, which is half of the 37-hour field failure - and every test of it so far has been against the in-process server, where "the node left" is a method call. Measured against a live three-node cluster: node 2 removed, and the client's endpoint for it disappeared while node 3's appeared in its place. cluster nodes: 1=master@63.35.180.156, 2=slave@54.228.99.95, 3=slave@34.245.21.12 endpoints before: , 63.35.180.156, 54.228.99.95 removing node 2 (we are served by node 1) +35.1s restored, +40.0s SocketClosed x2, +40.1s restored, +54.1s action finished endpoints after: , 63.35.180.156, 34.245.21.12 Reads and writes either side were unaffected, which is the part that matters. Node discovery comes from the injector, not the cluster's own API: port 9443 is not reachable from outside the deployment's network (a socket error, not an authentication one), so `execute_rladmin_command` running `status nodes` cluster-side is the portable source of truth. Text parsing is not lovely, but hardcoding node ids is what made the first destructive run prove nothing. Two parameter shapes learned by being told: execute_rladmin_command wants `bdb_id` *and* `rladmin_command`, and node_remove is node-scoped like node_failure. Also upgrades the earlier node_failure case to target the node actually serving the database, rather than whichever one was guessed. --- .../DestructiveScenarioTests.cs | 89 +++++++++++++ .../Environment/ClusterNodes.cs | 102 +++++++++++++++ .../Environment/ClusterRestClient.cs | 1 + .../NodeRemovalScenarioTests.cs | 122 ++++++++++++++++++ 4 files changed, 314 insertions(+) create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs index 562b419e0..9b39e2393 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs @@ -51,6 +51,95 @@ public class DestructiveScenarioTests(ReplicatedDatabaseFixture fixture, ITestOu private static bool Enabled => string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase); + /// + /// Kills the node that actually serves this database, rather than an arbitrary one. + /// + /// + /// The version of the test below that means something. Measured 2026-09-03: node_failure against a + /// node we were not connected through produced zero connection drops - the deployment absorbed it and the + /// client never noticed - so a green run proved nothing about our recovery. Resolving the endpoint to a + /// node first is what makes the failure land where the client can see it. + /// + [Fact] + public async Task KillingTheNodeThatServesUsIsSurvived() + { + if (!Enabled) Assert.Skip($"set {EnableVariable}=true to run the destructive scenarios; they damage the cluster"); + + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + + var nodes = await ClusterNodes.ListAsync(fixture.Injector, database.BdbId, cancellationToken); + log.WriteLine($"cluster nodes: {string.Join(", ", nodes.Select(n => $"{n.Id}={n.Role}@{n.ExternalAddress}"))}"); + + var serving = await ClusterNodes.FindServingAsync(fixture.Injector, database.BdbId, database.Host, cancellationToken); + if (serving is null) Assert.Skip($"could not map {database.Host} to a node, so this would kill an arbitrary one"); + + log.WriteLine($"{database} is served by node {serving.Id} ({serving.Role}@{serving.ExternalAddress})"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + const string Key = "fi-node-failure-targeted"; + await db.StringSetAsync(Key, "before"); + + var drops = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (drops) drops.Add(e.FailureType); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId}"); + } + }; + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "node_failure", + new Dictionary { ["node_id"] = serving.Id.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not run 'node_failure' against node {serving.Id}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports node_failure finished"); + + var recovered = await Poll.UntilAsync( + () => + { + try + { + return db.StringGet(Key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 180_000, + pollMilliseconds: 1000); + + lock (drops) + { + log.WriteLine( + $" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered={recovered} after {drops.Count} drop(s): " + + (drops.Count == 0 ? "(none)" : string.Join(", ", drops.Distinct()))); + } + + Assert.True(recovered, "the client should recover on its own after the node serving it is killed"); + Assert.Equal("before", await db.StringGetAsync(Key)); + } + [Theory] [InlineData("shard_failure", "bdb_id")] [InlineData("proxy_failure", "bdb_id")] diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs new file mode 100644 index 000000000..6e81e1843 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The cluster's node list, read through the fault injector rather than the cluster's own API. +/// +/// +/// The management API on port 9443 is not reachable from outside the deployment's network - measured: a socket +/// error, not an authentication one - so anything a test needs to know about nodes has to come back the same +/// way it gives instructions. execute_rladmin_command runs cluster-side and returns stdout, which makes +/// status nodes the portable source of truth. +/// +/// Text parsing is not lovely, but it is honest about where the information comes from, and the alternative - +/// hardcoding node ids - is what made the first destructive run prove nothing. +/// +/// +internal static class ClusterNodes +{ + internal sealed record Node(int Id, string Role, string Address, string ExternalAddress); + + /// + /// Runs rladmin status nodes and returns what it said. + /// + /// + /// is required by the action even though the command is cluster-wide: the + /// injector resolves a database to decide where to run. + /// + public static async Task> ListAsync(FaultInjectorClient injector, int bdbId, CancellationToken cancellationToken) + { + var result = await injector.RunActionAsync( + "execute_rladmin_command", + new Dictionary + { + ["bdb_id"] = bdbId.ToString(), + ["rladmin_command"] = "status nodes", + }, + cancellationToken: cancellationToken).ConfigureAwait(false); + + // the action's payload nests the command's stdout under output.output + var text = result.ValueKind == JsonValueKind.Object + && result.TryGetProperty("output", out var inner) + && inner.ValueKind == JsonValueKind.Object + && inner.TryGetProperty("output", out var stdout) + ? stdout.GetString() + : null; + + return text is null ? [] : Parse(text); + } + + /// + /// Which node currently answers for this hostname, or null if the addresses do not match any node. + /// + /// + /// The step that makes a node-scoped fault mean anything: killing an arbitrary node usually proves + /// nothing, because the deployment absorbs it and the client never notices. + /// + public static async Task FindServingAsync( + FaultInjectorClient injector, + int bdbId, + string host, + CancellationToken cancellationToken) + { + var nodes = await ListAsync(injector, bdbId, cancellationToken).ConfigureAwait(false); + var addresses = (await Dns.GetHostAddressesAsync(host, cancellationToken).ConfigureAwait(false)) + .Select(a => a.ToString()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + return nodes.FirstOrDefault(n => + addresses.Contains(n.ExternalAddress) || addresses.Contains(n.Address)); + } + + /// + /// Reads the fixed-column output of status nodes. + /// + /// + /// The leading * marks the node the command ran against, so it is stripped rather than parsed. + /// + internal static List Parse(string output) + { + var nodes = new List(); + foreach (var line in output.Split('\n')) + { + var match = Regex.Match( + line.Trim(), + @"^\*?node:(?\d+)\s+(?\S+)\s+(?\S+)\s+(?\S+)"); + if (match.Success && int.TryParse(match.Groups["id"].Value, out var id)) + { + nodes.Add(new Node(id, match.Groups["role"].Value, match.Groups["addr"].Value, match.Groups["ext"].Value)); + } + } + + return nodes; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs index c9565bb79..8cddd2841 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs @@ -68,4 +68,5 @@ public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials return results; } + } diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs new file mode 100644 index 000000000..3deeadc17 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// An OSS cluster API database, so the client holds one endpoint per node and can retire one. +public sealed class OssClusterLifecycleFixture() : FaultInjectorFixture(DatabaseShape.OssClusterApi); + +/// +/// A node removed from the cluster: endpoint retirement measured against a real deployment rather than a fake. +/// +/// +/// The one destructive scenario that exercises something this feature built. Pruning exists because the +/// endpoint collection used to be add-only: a node that left the cluster was dialled forever, which is half of +/// the 37-hour field failure. Every test of it so far has been against the in-process server, where "the node +/// left" is a method call; here the node genuinely leaves, the topology genuinely changes, and the endpoint has +/// to be let go without dropping the caller's work. +/// +/// Needs the OSS cluster API shape. A proxied standalone database is reached through one hostname however many +/// nodes are behind it, so there is no per-node endpoint to retire and the test would be vacuous. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "destructive")] +public class NodeRemovalScenarioTests(OssClusterLifecycleFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string EnableVariable = "SER_FI_DESTRUCTIVE"; + + private static bool Enabled => + string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase); + + [Fact] + public async Task RemovingANodeRetiresItsEndpoint() + { + if (!Enabled) Assert.Skip($"set {EnableVariable}=true to run the destructive scenarios; they damage the cluster"); + + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + + var nodes = await ClusterNodes.ListAsync(fixture.Injector, database.BdbId, cancellationToken); + log.WriteLine($"cluster nodes: {string.Join(", ", nodes.Select(n => $"{n.Id}={n.Role}@{n.ExternalAddress}"))}"); + if (nodes.Count < 3) Assert.Skip($"only {nodes.Count} node(s); removing one needs somewhere for its shards to go"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + const string Key = "fi-node-remove"; + await db.StringSetAsync(Key, "before"); + + conn.ConnectionFailed += (_, e) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId}"); + } + }; + + var before = conn.GetEndPoints(); + log.WriteLine($"endpoints before: {string.Join(", ", before.Select(e => e.ToString()))}"); + + // The node whose endpoint we hold *and* which is not serving our own connection, so the removal is + // visible as a retirement rather than as a reconnect. If we cannot tell them apart, any node we hold + // an endpoint for will do - the retirement is the assertion either way. + var serving = await ClusterNodes.FindServingAsync(fixture.Injector, database.BdbId, database.Host, cancellationToken); + + // not the node serving us, and not node 1: cluster management lives there on a default install, and + // taking it out takes the fault injector's own access with it + var candidate = nodes.FirstOrDefault(n => n.Id != serving?.Id && n.Id != 1 && n.Role != "master"); + if (candidate is null) Assert.Skip("no node that is safe to remove and observable from here"); + + log.WriteLine($"removing node {candidate.Id} ({candidate.ExternalAddress}); we are served by node {serving?.Id.ToString() ?? "(unknown)"}"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "node_remove", + new Dictionary { ["node_id"] = candidate.Id.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not remove node {candidate.Id}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports node_remove finished"); + + // Retirement needs several consecutive topology passes that do not list the node, and those are driven + // by the config-check interval, so this is tens of seconds rather than immediate by design. + var retired = await Poll.UntilAsync( + () => !conn.GetEndPoints().Any(e => e.ToString()!.Contains(candidate.ExternalAddress, StringComparison.OrdinalIgnoreCase)), + timeoutMilliseconds: 180_000, + pollMilliseconds: 2000); + + log.WriteLine($"endpoints after: {string.Join(", ", conn.GetEndPoints().Select(e => e.ToString()))}"); + + // The caller's work is the part that must not suffer, whatever the endpoint collection does. + Assert.Equal("before", await db.StringGetAsync(Key)); + await db.StringSetAsync(Key, "after"); + Assert.Equal("after", await db.StringGetAsync(Key)); + + if (!retired) + { + // Reported rather than asserted: the endpoint set a proxied cluster advertises does not have to + // name every node, so "the address never appeared in our endpoints" is a legitimate outcome and + // not a pruning failure. The log above says which it was. + log.WriteLine( + $" note: {candidate.ExternalAddress} was not present in our endpoint set, or was not retired within the bound; " + + "traffic was unaffected either way"); + } + } +} From f7b1f97069c41db93ff0b8a7a7a6d067517d557b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 16:03:53 +0100 Subject: [PATCH 33/36] cluster_failure ends the deployment; assert clean degradation, not recovery Ran it, and the first version of this test was asking the wrong question. The action takes a `node_ids` list, stops those nodes, and restores nothing - so with every node named, the cluster stays down: rladmin stops answering and the environment needs re-provisioning. The client reported one SocketClosed at +1.7s and never recovered, across 300 seconds of polling, which is correct rather than a defect: there was nothing to recover to. So the assertion is now what a client can actually be held to under a total outage - the failure is observed, and every subsequent command fails as a Redis-family exception rather than hanging, crashing, or throwing something a caller could not have written a catch block for. If the deployment does come back, coming back with it is still asserted. Honest about verification: the previous shape was run live and is what produced these findings, but the rewritten assertions have not been re-run, because running them needs a cluster and this scenario is what removed it. Also records every parameter shape learned this session in the tier README, since exception messages are the only documentation of them, and notes that node discovery goes through the injector because the cluster's REST API on 9443 is not reachable from outside its network. --- .../ClusterWideFailureScenarioTests.cs | 142 ++++++++++++++++++ .../README.md | 18 ++- 2 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs new file mode 100644 index 000000000..fafc0c8a6 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The whole cluster goes away, and does not come back. +/// +/// +/// Gated separately from the other destructive scenarios, and deliberately so: killing one node damages one +/// node, while this takes out every database on the cluster including anybody else's. It is the last thing you +/// run against a deployment, and **it ends the deployment** - measured 2026-09-03: cluster_failure +/// takes a node_ids list, stops those nodes, and restores nothing. With every node in the list the +/// cluster stays down; rladmin stops answering, and the environment needs re-provisioning. +/// +/// So recovery is not assertable here, and the first version of this test was wrong to try: it asserted the +/// client would come back, which it cannot do when there is nothing to come back to. What *is* assertable is +/// that a total outage degrades cleanly - one reported failure, then commands that fail as Redis exceptions +/// rather than hanging, crashing, or throwing something unrelated - and that the multiplexer stays in a state +/// where it would recover if the deployment did. +/// +/// +/// reset_cluster is deliberately *not* here. It rebuilds the cluster from scratch, so the databases a +/// client was using cease to exist and there is no recovery to observe: it is a lifecycle operation for +/// whoever owns the environment, not a client test, and writing it as one would only produce a test that +/// asserts nothing. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "destructive")] +public class ClusterWideFailureScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string EnableVariable = "SER_FI_CLUSTER_FAILURE"; + + [Fact] + public async Task ATotalOutageFailsCleanlyRatherThanWedging() + { + if (!string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase)) + { + Assert.Skip($"set {EnableVariable}=true to run this; it takes out every database on the cluster, not just ours"); + } + + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + log.WriteLine($"cluster_failure against {database}"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + const string Key = "fi-cluster-failure"; + await db.StringSetAsync(Key, "before"); + + var drops = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (drops) drops.Add(e.FailureType); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + + // Every node, together: the action is "fail these nodes", not "fail the cluster" - it wants a + // `node_ids` list, which the schema does not say and which an exception was kind enough to tell us. + var nodes = await ClusterNodes.ListAsync(fixture.Injector, database.BdbId, cancellationToken); + log.WriteLine($"cluster nodes: {string.Join(", ", nodes.Select(n => $"{n.Id}={n.Role}@{n.ExternalAddress}"))}"); + if (nodes.Count == 0) Assert.Skip("no nodes could be listed, so there is nothing to fail"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "cluster_failure", + new Dictionary + { + ["bdb_id"] = database.BdbId.ToString(), + ["node_ids"] = nodes.Select(n => n.Id).ToArray(), + }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not run 'cluster_failure': {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports cluster_failure finished"); + + // Probe for a while and record *how* it fails. Recovery is not expected - see the remarks - so the + // interesting question is whether every failure is a Redis-family one, which is what a caller can + // write a catch block for. + var faults = new List(); + var recovered = false; + var deadline = clock.Elapsed + TimeSpan.FromSeconds(120); + while (clock.Elapsed < deadline && !recovered) + { + try + { + recovered = db.StringGet(Key) == "before"; + } + catch (Exception ex) + { + lock (faults) faults.Add(ex.GetType().Name); + } + + await Task.Delay(2000, cancellationToken); + } + + lock (drops) + { + log.WriteLine( + $" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered={recovered} after {drops.Count} drop(s): " + + (drops.Count == 0 ? "(none)" : string.Join(", ", drops.Distinct()))); + } + + string[] observedFaults; + lock (faults) observedFaults = [.. faults.Distinct()]; + log.WriteLine($" faults: {(observedFaults.Length == 0 ? "(none)" : string.Join(", ", observedFaults))}"); + + if (recovered) + { + // If the deployment does come back - a smaller node list, or somebody restarting it - then coming + // back with it is the requirement, so say so rather than passing silently. + Assert.Equal("before", await db.StringGetAsync(Key)); + return; + } + + lock (drops) Assert.NotEmpty(drops); // the outage has to have been observed, or this proves nothing + Assert.NotEmpty(observedFaults); + Assert.All(observedFaults, name => Assert.Contains(name, new[] + { + nameof(RedisConnectionException), + nameof(RedisTimeoutException), + nameof(RedisServerException), + nameof(TimeoutException), + })); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index a381b1acc..3461932fd 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -60,8 +60,22 @@ cluster's life, watching. Two things measured on 2026-09-03 that change how you read a green run: -- **The scope differs per action**, and the injector's schema does not say so: `shard_failure` and - `proxy_failure` take `bdb_id`, `node_failure` takes `node_id` and rejects a `bdb_id` outright. +- **The scope differs per action**, and the injector's schema does not say so. Learned by being told: + + | action | parameter | + |---|---| + | `shard_failure`, `proxy_failure` | `bdb_id` | + | `node_failure`, `node_remove` | `node_id` | + | `cluster_failure` | `node_ids` (a list) | + | `execute_rladmin_command` | `bdb_id` *and* `rladmin_command` | + +- **`SER_FI_CLUSTER_FAILURE=true` is a separate gate, and it ends the deployment.** `cluster_failure` stops + the nodes you name and restores nothing, so naming them all leaves the cluster down for good - `rladmin` + stops answering and the environment needs re-provisioning. Run it last or not at all. +- **Node-scoped actions need the *right* node.** `ClusterNodes.FindServingAsync` resolves the database + hostname and matches it against `rladmin status nodes`, because killing an arbitrary node usually proves + nothing: the deployment absorbs it and the client never notices. Node discovery goes through the injector + rather than the cluster's REST API on 9443, which is not reachable from outside the deployment's network. - **Only `proxy_failure` was visible to the client** (one `SocketClosed`, restored ~8s later). `shard_failure` on a replicated database and `node_failure` against a node we were not connected through both produced zero drops - the deployment absorbed them. That is worth knowing and is *not* coverage of our recovery path, so From 1a4c8587f6b79db3a1e5e57ff4f3527919ed412c Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 3 Sep 2026 16:13:56 +0100 Subject: [PATCH 34/36] Warn that a stale environment directory passes every check Learned the hard way: the credentials check only verifies env_output.json is well-formed, and node discovery and provisioning go through the injector, so a directory describing a cluster that died a week ago runs happily until something reaches for the template databases. --- tests/StackExchange.Redis.FaultInjector.Tests/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index 3461932fd..a3a6f2356 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -15,6 +15,12 @@ export E2E_SCENARIO_TESTS=true # explicit opt-in: these create and dele dotnet test tests/StackExchange.Redis.FaultInjector.Tests ``` +**Point it at the *current* environment.** A stale directory passes every check we make: the credentials are +only checked for being well-formed, and node discovery and database provisioning both go through the injector, +which is configured separately - so the only thing that fails is the template databases, which belong to a +cluster that no longer exists. If `endpoints.json` names a different cluster than the one the injector is +driving, that directory is not the one you want. + One path is the whole configuration. That directory is the one mounted into the injector as `/app/config`, so it already holds the cluster credentials (`env_output.json`), the CA certificate, and the compose file; nothing has to be hand-carried into the test run. `FAULT_INJECTION_API_URL` overrides the injector URL From 16adfe35f12f2351dd4b49db9f92ca60df5e6d7c Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 8 Sep 2026 11:10:14 +0100 Subject: [PATCH 35/36] Re-read the topology on a long timer as a backstop The third finding from the 37-hour field failure, and the only one still open: there is no periodic topology refresh. The other two are closed - an endpoint that only ever refuses now provokes a refresh after three consecutive connect failures, and a node that has left the topology is retired rather than dialled forever - but between them they still need *something* to go wrong. What nothing covers is an endpoint that is reachable, completes a handshake, and is no longer part of the deployment: a re-bound port now serving something else produces no failure, no redirect and nothing announced, so no event-driven path asks the question. That was invisible for the lifetime of the multiplexer. topologyRefreshSeconds re-reads it anyway, every 30 minutes by default, or never if set to zero. Two things keep the cost honest: the interval is long, and each client picks its own phase within a hard-coded 30-second jitter on every cycle, so a fleet started together does not stay in step. Beyond that it is the ordinary refresh path, which already declines while another is in flight. The first interval runs from the first heartbeat rather than from construction, so a multiplexer that is created, used and disposed inside it costs nothing. Deliberately a backstop rather than the mechanism: the event-driven paths react in seconds where this reacts in minutes, and this is the one refresh whose cost is paid on a schedule rather than in response to something. Note this is a new default for every consumer, not only those using maintenance notifications, and it is not gated behind the experiment: it is ordinary topology hygiene rather than part of that feature. ConfigTests.ExpectedFields earned its keep here - it caught the new field being added without Clone() knowing about it, which would have silently dropped the setting from any cloned configuration. The test only checks that somebody looked, so the round-trip is now asserted directly too. --- docs/Configuration.md | 17 +++ .../Configuration/DefaultOptionsProvider.cs | 10 ++ .../ConfigurationOptions.cs | 36 +++++ .../ConnectionMultiplexer.cs | 53 +++++++ .../PublicAPI/PublicAPI.Unshipped.txt | 3 + .../StackExchange.Redis.Tests/ConfigTests.cs | 1 + .../PeriodicTopologyRefreshTests.cs | 135 ++++++++++++++++++ 7 files changed, 255 insertions(+) create mode 100644 tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs diff --git a/docs/Configuration.md b/docs/Configuration.md index 80dc94cf6..dc63dfda4 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -78,6 +78,7 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | connectTimeout={int} | `ConnectTimeout` | `5000` | Timeout (ms) for connect operations | | configChannel={string} | `ConfigurationChannel` | `__Booksleeve_MasterChanged` | Broadcast channel name for communicating configuration changes | | configCheckSeconds={int} | `ConfigCheckSeconds` | `60` | Time (seconds) to check configuration. This serves as a keep-alive for interactive sockets, if it is supported. | +| topologyRefreshSeconds={int} | `TopologyRefreshSeconds` | `1800` | Time (seconds) between unprompted topology re-reads, or `0` to never do so. Jittered by up to 30 seconds. | | defaultDatabase={int} | `DefaultDatabase` | `null` | Default database index, from `0` to `databases - 1` | | keepAlive={int} | `KeepAlive` | `-1` | Time (seconds) at which to send a message to help keep sockets alive (60 sec default) | | tcpKeepAlive={bool} | `TcpKeepAlive` | `true` | Enables TCP keep-alive when appropriate (endpoint- and platform-dependent) | @@ -288,6 +289,22 @@ Note that `configCheckSeconds` on its own is *not* a periodic topology refresh - `INFO replication` on an established connection, which is a replication-role check. This is the path that notices an endpoint nobody can reach. +### ...and the backstop for one nobody can fault + +Repeated connect failures cover an endpoint that refuses or never finishes a handshake. What they cannot cover +is an endpoint that is *reachable*, answers a handshake, and is no longer part of the deployment: a re-bound +port now serving something else produces no failure, no redirect, and nothing announced, so no event-driven +path asks the question. + +`topologyRefreshSeconds` is the answer to that, and only that: every 30 minutes by default, the client +re-reads the topology whether or not anything appears to be wrong. Two things keep it cheap. The interval is +long, and each client picks its own phase within a 30-second jitter on every cycle, so a fleet started +together does not stay in step. Set it to `0` to turn it off. + +It is deliberately a backstop rather than the mechanism. Topology is normally learned from something +happening - a redirect, an announcement, a maintenance notification, a connection failing - and those react in +seconds where this reacts in minutes. + ## ReconnectRetryPolicy StackExchange.Redis automatically tries to reconnect in the background when the connection is lost for any reason. It keeps retrying until the connection has been restored. It would use ReconnectRetryPolicy to decide how long it should wait between the retries. diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index fb5f88c66..65353ddf2 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -283,6 +283,16 @@ public static DefaultOptionsProvider GetProvider(EndPoint endpoint) /// public virtual TimeSpan ConfigCheckInterval => TimeSpan.FromMinutes(1); + /// + /// Gets how often to re-read the deployment's topology when nothing has gone wrong, or + /// to never do so. + /// + /// + /// Long by design: this is a backstop for a topology change that produced no failure, no redirect and + /// no announcement, and its cost is paid by every client on the schedule at once. + /// + public virtual TimeSpan TopologyRefreshInterval => TimeSpan.FromMinutes(30); + /// /// The username to use to authenticate with the server. /// diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 0ef37e53e..ae46ac359 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -151,6 +151,7 @@ internal const string ChannelPrefix = "channelPrefix", ConfigChannel = "configChannel", ConfigCheckSeconds = "configCheckSeconds", + TopologyRefreshSeconds = "topologyRefreshSeconds", ConnectRetry = "connectRetry", ConnectTimeout = "connectTimeout", DefaultDatabase = "defaultDatabase", @@ -195,6 +196,7 @@ internal const string ClientName, ConfigChannel, ConfigCheckSeconds, + TopologyRefreshSeconds, ConnectRetry, ConnectTimeout, DefaultDatabase, @@ -287,6 +289,7 @@ private enum OptionFlags : ulong MaintenanceRelaxedTimeoutHasValue = 1UL << 37, MaintenanceRelaxedWindowMaxHasValue = 1UL << 38, MaintenancePostEventRelaxedDurationHasValue = 1UL << 39, + TopologyRefreshSecondsHasValue = 1UL << 41, } private OptionFlags optionFlags; @@ -300,6 +303,7 @@ private enum OptionFlags : ulong private Version? defaultVersion; private int keepAlive, asyncTimeout, syncTimeout, connectTimeout, responseTimeout, connectRetry, configCheckSeconds, defaultDatabase; + private int topologyRefreshSeconds; private Proxy proxy; @@ -1022,6 +1026,33 @@ public int ConfigCheckSeconds set => SetWithValue(OptionFlags.ConfigCheckSecondsHasValue, ref configCheckSeconds, value); } + /// + /// Re-read the deployment's topology every n seconds even when nothing has gone wrong, or 0 to + /// never do so (every 30 minutes by default). + /// + /// + /// A backstop, not the primary mechanism. Topology is normally learned from something happening: a + /// redirect from a reachable node, a configuration announcement, a maintenance notification, or a + /// connection failing. Each of those needs *somebody* to notice, and the case none of them covers is an + /// endpoint that is reachable, answers a handshake, and is no longer part of the deployment - no + /// failure, no redirect, nothing announced. + /// + /// The default is deliberately long, and each refresh is spread by up to 30 seconds of jitter, because + /// the cost of this is paid per client: a fleet of them re-reading configuration on the same schedule + /// is exactly the stampede that the failure-driven paths are careful to avoid. If you want it off, set + /// it to zero. + /// + /// + /// Distinct from , which despite its name does not re-read topology - + /// it sends an INFO replication on an established connection to check a server's role. + /// + /// + public int TopologyRefreshSeconds + { + get => HasValue(OptionFlags.TopologyRefreshSecondsHasValue) ? topologyRefreshSeconds : (int)Defaults.TopologyRefreshInterval.TotalSeconds; + set => SetWithValue(OptionFlags.TopologyRefreshSecondsHasValue, ref topologyRefreshSeconds, value); + } + /// /// Parse the configuration from a comma-delimited configuration string. /// @@ -1071,6 +1102,7 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow #pragma warning restore CS0618 // Type or member is obsolete connectRetry = connectRetry, configCheckSeconds = configCheckSeconds, + topologyRefreshSeconds = topologyRefreshSeconds, responseTimeout = responseTimeout, defaultDatabase = defaultDatabase, reconnectRetryPolicy = reconnectRetryPolicy, @@ -1177,6 +1209,7 @@ public string ToString(bool includePassword) Append(sb, OptionKeys.ConnectRetry, OptionFlags.ConnectRetryHasValue, in connectRetry); Append(sb, OptionKeys.Proxy, OptionFlags.ProxyHasValue, in proxy); Append(sb, OptionKeys.ConfigCheckSeconds, OptionFlags.ConfigCheckSecondsHasValue, in configCheckSeconds); + Append(sb, OptionKeys.TopologyRefreshSeconds, OptionFlags.TopologyRefreshSecondsHasValue, in topologyRefreshSeconds); Append(sb, OptionKeys.ResponseTimeout, OptionFlags.ResponseTimeoutHasValue, in responseTimeout); Append(sb, OptionKeys.DefaultDatabase, OptionFlags.DefaultDatabaseHasValue, in defaultDatabase); Append(sb, OptionKeys.SetClientLibrary, OptionFlags.SetClientLibraryHasValue, OptionFlags.SetClientLibraryValue); @@ -1390,6 +1423,9 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) case OptionKeys.ConnectRetry: ConnectRetry = OptionKeys.ParseInt32(key, value); break; + case OptionKeys.TopologyRefreshSeconds: + TopologyRefreshSeconds = OptionKeys.ParseInt32(key, value); + break; case OptionKeys.ConfigCheckSeconds: ConfigCheckSeconds = OptionKeys.ParseInt32(key, value); break; diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index d4182cf41..3a5261298 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1332,6 +1332,57 @@ public void UnRoot(int token) } } + private int _nextTopologyRefreshTicks; // 0 until the first heartbeat schedules one + + /// How far apart two clients' refreshes are spread; not configurable, because nobody needs to tune it. + private const int TopologyRefreshJitterMilliseconds = 30_000; + + /// + /// Re-reads the topology on a long timer, as a backstop for a change that nothing reported. + /// + /// + /// Every other refresh path is event-driven: a redirect, an announcement, a notification, or a + /// connection failing. They cover almost everything between them - the case left over is an endpoint + /// that is reachable, completes a handshake, and is no longer part of the deployment, which produces + /// none of those signals and so was previously invisible for the lifetime of the multiplexer. + /// + /// Two things keep the cost honest. The interval is long (30 minutes by default), and each client + /// picks its own phase within a 30-second jitter on every cycle, so a fleet started together does not + /// stay in step. Beyond that this is the ordinary refresh path, which already declines while another + /// is in flight. + /// + /// + /// The first interval is measured from the first heartbeat rather than from construction, so nothing + /// is read on behalf of a multiplexer that is created, used briefly and disposed. + /// + /// + private void CheckTopologyRefreshDue(int now) + { + var seconds = RawConfig.TopologyRefreshSeconds; + if (seconds <= 0 || _isDisposed) return; + + var next = Volatile.Read(ref _nextTopologyRefreshTicks); + if (next == 0) + { + Interlocked.CompareExchange(ref _nextTopologyRefreshTicks, ScheduleTopologyRefresh(now, seconds), 0); + return; + } + + if (unchecked(now - next) < 0) return; // not due yet + + // reschedule *before* refreshing, and only if nobody else got there first: a refresh that takes + // longer than a heartbeat must not queue a second one behind it + if (Interlocked.CompareExchange(ref _nextTopologyRefreshTicks, ScheduleTopologyRefresh(now, seconds), next) != next) return; + + ReconfigureIfNeeded(null, fromBroadcast: false, "periodic topology refresh"); + } + + private static int ScheduleTopologyRefresh(int now, int seconds) + { + var due = unchecked(now + (seconds * 1000) + ServerSelectionStrategy.SharedRandom.Next(TopologyRefreshJitterMilliseconds)); + return due == 0 ? 1 : due; // zero means "not scheduled", so never land on it + } + internal void OnHeartbeat() { try @@ -1341,6 +1392,8 @@ internal void OnHeartbeat() Interlocked.Exchange(ref lastGlobalHeartbeatTicks, now); Trace("heartbeat"); + CheckTopologyRefreshDue(now); + var tmp = GetServerSnapshot(); int token = 0; bool isRooted = pulse?.IsRooted(out token) ?? false, hasPendingCallerFacingItems = false; diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index ac812c230..c7261f397 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -164,6 +164,8 @@ override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Protocol.ge override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.ConfigurationChannel.get -> string! override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Name.get -> string! override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +StackExchange.Redis.ConfigurationOptions.TopologyRefreshSeconds.get -> int +StackExchange.Redis.ConfigurationOptions.TopologyRefreshSeconds.set -> void StackExchange.Redis.ProductVariant.Dragonfly = 3 -> StackExchange.Redis.ProductVariant StackExchange.Redis.ProductVariant.Memurai = 4 -> StackExchange.Redis.ProductVariant StackExchange.Redis.ProductVariant.Redict = 5 -> StackExchange.Redis.ProductVariant @@ -191,3 +193,4 @@ static StackExchange.Redis.BitFieldOperation.Set(StackExchange.Redis.BitFieldEnc static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.Name.get -> string? +virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.TopologyRefreshInterval.get -> System.TimeSpan diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 4a1a3dba9..e0cbb8c0a 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -111,6 +111,7 @@ orderby name "sslProtocols", "syncTimeout", "tieBreaker", + "topologyRefreshSeconds", "Tunnel", "user", }, diff --git a/tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs b/tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs new file mode 100644 index 000000000..d665d45e7 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Re-reading the topology on a timer, as a backstop for a change nothing reported. +/// +/// +/// The last of the three findings from the 37-hour field failure. The other two are closed: an endpoint that +/// only ever refuses now provokes a refresh after three consecutive connect failures, and a node that has left +/// the topology is retired rather than dialled forever. What neither covers is an endpoint that is +/// *reachable*, completes a handshake, and is no longer part of the deployment - it produces no failure, no +/// redirect and no announcement, so nothing asks the question. +/// +/// Non-parallel: these drive the multiplexer's own heartbeat and assert on what the server received, which a +/// neighbouring test's traffic would confuse. +/// +/// +[Collection(NonParallelCollection.Name)] +public class PeriodicTopologyRefreshTests(ITestOutputHelper log) +{ + /// Counts inbound CLUSTER commands, so a refresh is visible from the server's side. + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0 && string.Equals(request.GetString(0), "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + + return base.Execute(client, in request); + } + } + + private static async Task<(CountingServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + int topologyRefreshSeconds) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.TopologyRefreshSeconds = topologyRefreshSeconds; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + [Fact] + public async Task TheTopologyIsReReadOnTheInterval() + { + // One second plus up to 30 of jitter, so this waits out the jitter rather than the interval: the + // interval is the part under test, and the jitter is what stops a fleet moving in step. + var (server, conn) = await ConnectAsync(log, topologyRefreshSeconds: 1); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + log.WriteLine($"cluster commands after connect: {before}"); + + Assert.True( + await Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 40_000, pollMilliseconds: 250), + "the topology should have been re-read without anything going wrong"); + + log.WriteLine($"cluster commands after the interval: {server.ClusterCommands}"); + } + } + + [Fact] + public async Task ZeroTurnsItOff() + { + // The escape hatch has to work, because this is the one refresh path whose cost is paid on a schedule + // rather than in response to something. Anybody who does not want it must be able to say so. + var (server, conn) = await ConnectAsync(log, topologyRefreshSeconds: 0); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + await Task.Delay(3000); + + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task NothingIsReadBeforeTheFirstIntervalElapses() + { + // The interval runs from the first heartbeat, not from construction, so a multiplexer that is created, + // used and disposed inside it costs nothing. Asserted because the natural implementation - schedule at + // construction - makes short-lived multiplexers pay for a feature they never benefit from. + var (server, conn) = await ConnectAsync(log, topologyRefreshSeconds: 3600); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + await Task.Delay(3000); + + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public void TheDefaultIsLongAndSurvivesTheConfigurationString() + { + // 30 minutes: long enough that the fleet cost is negligible, short enough to bound how long a stale + // view can persist when nothing else notices. + Assert.Equal(1800, new ConfigurationOptions().TopologyRefreshSeconds); + + var parsed = ConfigurationOptions.Parse("localhost,topologyRefreshSeconds=120"); + Assert.Equal(120, parsed.TopologyRefreshSeconds); + Assert.Contains("topologyRefreshSeconds=120", parsed.ToString()); + + // ...and an explicit zero has to round-trip, or turning it off in a configuration string would look + // like it worked while silently reverting to the default + var disabled = ConfigurationOptions.Parse("localhost,topologyRefreshSeconds=0"); + Assert.Equal(0, disabled.TopologyRefreshSeconds); + Assert.Contains("topologyRefreshSeconds=0", disabled.ToString()); + + // Clone has to carry it too. ConfigTests.ExpectedFields is the "have you considered?" guard for this + // and it did its job - it caught the field being added without Clone knowing about it - but it checks + // that somebody looked, not that they got it right, so assert the behaviour as well. + Assert.Equal(0, disabled.Clone().TopologyRefreshSeconds); + Assert.Equal(120, parsed.Clone().TopologyRefreshSeconds); + } +} From 377abfad04b485ec7486b61b529c833d33e9963b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 8 Sep 2026 11:34:42 +0100 Subject: [PATCH 36/36] Say what ConfigCheckSeconds actually checks All three descriptions of it were some variant of "check configuration every n seconds", which says nothing about what is checked and reads as a topology re-read - the one thing it is not. That was tolerable while it was the only setting of its kind and is not now that topologyRefreshSeconds sits next to it. What it does: an INFO replication on each *established* interactive connection, which is how a primary/replica change is noticed on a deployment that does not announce one, and which doubles as the keep-alive for those sockets - hence the one-minute default. It says nothing about servers we cannot reach, endpoints that have left the deployment, or cluster slot ownership. Also documents that zero disables it, which none of the three mentioned and which the heartbeat has always honoured. --- docs/Configuration.md | 2 +- .../Configuration/DefaultOptionsProvider.cs | 7 ++++++- src/StackExchange.Redis/ConfigurationOptions.cs | 14 +++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index dc63dfda4..cc5789aa8 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -77,7 +77,7 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | connectRetry={int} | `ConnectRetry` | `3` | The number of times to repeat connect attempts during initial `Connect` | | connectTimeout={int} | `ConnectTimeout` | `5000` | Timeout (ms) for connect operations | | configChannel={string} | `ConfigurationChannel` | `__Booksleeve_MasterChanged` | Broadcast channel name for communicating configuration changes | -| configCheckSeconds={int} | `ConfigCheckSeconds` | `60` | Time (seconds) to check configuration. This serves as a keep-alive for interactive sockets, if it is supported. | +| configCheckSeconds={int} | `ConfigCheckSeconds` | `60` | Time (seconds) between re-checks of each connected server's replication role, via `INFO replication`; also acts as a keep-alive for interactive sockets. Not a topology re-read: see `topologyRefreshSeconds` | | topologyRefreshSeconds={int} | `TopologyRefreshSeconds` | `1800` | Time (seconds) between unprompted topology re-reads, or `0` to never do so. Jittered by up to 30 seconds. | | defaultDatabase={int} | `DefaultDatabase` | `null` | Default database index, from `0` to `databases - 1` | | keepAlive={int} | `KeepAlive` | `-1` | Time (seconds) at which to send a message to help keep sockets alive (60 sec default) | diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index 65353ddf2..055c3d23d 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -279,8 +279,13 @@ public static DefaultOptionsProvider GetProvider(EndPoint endpoint) public virtual string TieBreaker => "__Booksleeve_TieBreak"; /// - /// Check configuration every n interval. + /// Gets how often to re-check the replication role of each connected server, or + /// to never do so. /// + /// + /// An INFO replication on each established interactive connection, which also serves as the + /// keep-alive for those sockets. Not a topology re-read - see . + /// public virtual TimeSpan ConfigCheckInterval => TimeSpan.FromMinutes(1); /// diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index ae46ac359..3fd822ea3 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -1018,8 +1018,20 @@ internal RemoteCertificateValidationCallback? CertificateValidationCallback } /// - /// Check configuration every n seconds (every minute by default). + /// How often to re-check the replication role of each connected server, in seconds (every minute by + /// default), or 0 to never do so. /// + /// + /// Sends an INFO replication on each *established* interactive connection. That is how a + /// primary/replica change is noticed on a deployment that does not announce one, and it doubles as the + /// keep-alive for those sockets, which is why the interval is short. + /// + /// Despite the name, this is not a topology re-read. It asks a server we are already talking to what it + /// says about itself, so it reveals nothing about servers we cannot reach, about endpoints that have + /// left the deployment, or about cluster slot ownership. is the + /// setting for that, and is deliberately much less frequent because it costs much more. + /// + /// public int ConfigCheckSeconds { get => HasValue(OptionFlags.ConfigCheckSecondsHasValue) ? configCheckSeconds : (int)Defaults.ConfigCheckInterval.TotalSeconds;