diff --git a/Directory.Build.props b/Directory.Build.props index f5981dc8b..c0863db83 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,7 +11,7 @@ $(MSBuildThisFileDirectory)Shared.ruleset NETSDK1069 - $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009 + $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009;SER010 https://github.com/StackExchange/StackExchange.Redis/releases https://seredis.dev/ MIT diff --git a/docs/Configuration.md b/docs/Configuration.md index 66eaa20ab..0c9afc388 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -100,6 +100,11 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | setlib={bool} | `SetClientLibrary` | `true` | Whether to attempt to use `CLIENT SETINFO` to set the library name/version on the connection | | protocol={string} | `Protocol` | `null` | Redis protocol to use; see section below | | highIntegrity={bool} | `HighIntegrity` | `false` | High integrity (incurs overhead) sequence checking on every command; see section below | +| defaults={string} | `Defaults` | `null` | Selects a named defaults provider; see section below | +| maintNotifications={string} | `MaintenanceNotifications` | `disabled` | Whether to ask servers for maintenance notifications; see section below | +| maintRelaxedTimeout={int} | `MaintenanceRelaxedTimeout` | `10` | **Seconds** that command timeouts are relaxed to during announced maintenance; see section below | +| maintRelaxedWindowMax={int} | `MaintenanceRelaxedWindowMax` | `30` | **Seconds** a relaxed window may last at most; see section below | +| maintPostEventRelaxed={int} | `MaintenancePostEventRelaxedDuration` | `20` | **Seconds** timeouts stay relaxed after maintenance completes; see section below | Additional code-only options: - LoggerFactory (`ILoggerFactory`) - Default: `null` @@ -287,6 +292,57 @@ config.ReconnectRetryPolicy = new LinearRetry(5000); //6 5000 ``` +## Defaults providers + +Some settings have sensible values that depend on *what you are connecting to* rather than on what you want, so +the library keeps them in a provider and consults it for anything you have not set explicitly. Providers are +normally chosen automatically by looking at the endpoints - an `*.redis.cache.windows.net` host selects the +Azure provider, and so on - and you can select one explicitly instead: + +``` +myserver:6379,defaults=enterprise +``` + +The names are `azure`, `amr` (Azure Managed Redis), `rediscloud` and `enterprise` (a self-managed Redis +Enterprise deployment). The last one exists because it cannot be detected: a self-managed cluster has whatever +DNS its operator gave it, so there is nothing to recognize. It is also the right choice for a hosted deployment +reached behind private DNS, a CNAME or a proxy, where the endpoint no longer looks like what it is. + +A provider chosen explicitly appears in `ToString()`; one that was merely inferred from the endpoints does not, +since writing it out would turn a guess into a decision. Custom providers (assigned in code, via +`ConfigurationOptions.Defaults`, or registered with `DefaultOptionsProvider.AddProvider`) work exactly as +before; they can additionally be named in a configuration string if they override `Name`. + +## Maintenance notifications + +Redis Enterprise and Redis Cloud can warn a connected client *before* a disruptive event - a shard migration, a +failover, or an endpoint being replaced - so the client can act ahead of it rather than discover it by way of a +broken connection. This requires RESP3, and the client asks for it per connection: + +| Mode | Behaviour | +| --- | --- | +| `disabled` | Never ask. The default, because most servers have never heard of the request. | +| `auto` | Ask, and carry on if the server refuses or the connection ends up RESP2. Safe against a mixture of servers, and what most callers want. | +| `enabled` | **Required**: ask, and reject the connection unless notifications are live. Only point this at a deployment you know supports them. | + +Note that `enabled` means *required*, not merely *on* - it is the cross-client name for that mode, and it will +refuse a connection that cannot deliver notifications, including any RESP2 connection. The `amr`, `rediscloud` +and `enterprise` providers select `auto` for you, so on those deployments you need not set anything. + +While a disruption has been announced, command timeouts are relaxed - raised to `maintRelaxedTimeout`, never +lowered, so a caller with a more generous timeout keeps it. The window ends when the server says the disruption +finished, and then stays relaxed for `maintPostEventRelaxed` longer, because completion is exactly when every +other client that received the same notification comes back. `maintRelaxedWindowMax` bounds a window whose +closing notification never arrives. Only *command* timeouts are affected: keep-alive and connection-failure +detection are untouched, so a server that dies mid-maintenance is still noticed on the usual schedule. + +These durations are in **seconds**, unlike every other timeout here, because those are the units the +cross-client specification names for them. + +Receiving a notification raises `ConnectionMultiplexer.ServerMaintenanceEvent` with a `PushMaintenanceEvent`, +and a timeout or connection fault that happened during an announced disruption carries a `MaintenanceType` +saying so. + ## Redis protocol RESP3 is a newer protocol (available on v6 servers and above) which allows (among other changes) pub/sub messages to be communicated on the *same* connection - which can be very diff --git a/docs/exp/SER010.md b/docs/exp/SER010.md new file mode 100644 index 000000000..2645b9393 --- /dev/null +++ b/docs/exp/SER010.md @@ -0,0 +1,58 @@ +Maintenance notifications ("smart client handoffs") are a server feature by which a server warns a +connected client in advance of a disruptive event - a shard migration, a failover, or the endpoint +itself being replaced - so the client can act before the disruption rather than react to a broken +connection afterwards. The client opts in per connection with `CLIENT MAINT_NOTIFICATIONS ON`, and the +notifications then arrive as RESP3 push frames on the connection that carries commands. + +The feature is experimental here for three separate reasons: + +1. **The server side is not universal.** Only Redis Enterprise and Redis Cloud emit these + notifications today; OSS Redis, Valkey and Garnet do not recognize the opt-in at all. That is why + `MaintenanceNotificationMode.Auto` exists, and why the default is `Disabled`. +2. **The wire contract is still moving.** Notification types have been proposed and withdrawn during + its development, the semantics of some fields (notably the sequence number) are not defined by any + specification, and no captured RESP transcript exists to validate against. Our parsing is + deliberately liberal, but the shapes may change. +3. **What the client *does* in response is the substantial part**, and it is being built in stages - + timeout relaxation, then endpoint handoff. Behaviour may therefore change materially between + versions while the diagnostic is in place, even where the API does not. + +`MaintenanceNotificationMode`: + +- `Disabled` (the default) - never ask; the server sends nothing. +- `Enabled` - **required**: ask, and *reject the connection* unless notifications end up live. That covers a + server that refuses, a server that answers `HELLO 3` as RESP2, and a configuration that could never ask in + the first place (`Protocol = Resp2`, or `HELLO` unavailable) - requiring a RESP3-only feature over RESP2 is + a contradiction, and it fails rather than being half-honoured. Only point this at a deployment you know + supports them. +- `Auto` - ask, and carry on if the server refuses or the connection ends up RESP2; the feature is then + simply off for that server and the connection is never rejected over it. Safe against a mixture of + servers, or one whose support you don't know, and what most callers want. + +The names are the cross-client ones (they match go-redis and redis-py, so a connection string ports +between clients), which makes `Enabled` easy to misread as "on" - it means "required". One deliberate +divergence: the cross-client spec only calls for interrupting the connection when the *server* errors, +whereas we extend that to the RESP2 cases above, on the grounds that a required feature which cannot +possibly be delivered is the same failure either way. + +## A deployment prerequisite worth knowing + +On Redis Enterprise there are *two* switches, and they are easy to conflate. A **cluster-level flag** decides +whether the subcommand exists at all - `client_maint_notifications` for proxy-routed databases, +`oss_cluster_client_maint_notifications` for `oss_cluster` ones - and the **per-connection opt-in** this option +controls asks one connection to receive them. A cluster running a supporting version with the flag off will +refuse `CLIENT MAINT_NOTIFICATIONS ON`, which `Auto` absorbs silently and `Enabled` turns into a refused +connection. So if `Enabled` is rejecting connections against a deployment you believe supports the feature, +check the cluster flag before suspecting the client. + +If you accept the above, you can suppress this warning by adding the following to your `csproj` file: + +```xml +$(NoWarn);SER010 +``` + +or more granularly / locally in C#: + +``` c# +#pragma warning disable SER010 +``` diff --git a/src/RESPite/Shared/Experiments.cs b/src/RESPite/Shared/Experiments.cs index 2cfcd94d6..7077164ad 100644 --- a/src/RESPite/Shared/Experiments.cs +++ b/src/RESPite/Shared/Experiments.cs @@ -21,6 +21,7 @@ internal static class Experiments public const string GeoRedundantFailover = "SER007"; public const string Server_8_10 = "SER008"; public const string Transport = "SER009"; + public const string MaintenanceNotifications = "SER010"; // ReSharper restore InconsistentNaming diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index d533d4506..e8b800521 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -39,11 +39,13 @@ public FaultContext(Exception fault) kind = RedisErrorKind.ConnectionFault; flags = connection.Flags; status = connection.CommandStatus; + MaintenanceType = connection.MaintenanceType; break; case RedisTimeoutException timeout: kind = RedisErrorKind.Timeout; flags = timeout.Flags; status = timeout.Commandstatus; + MaintenanceType = timeout.MaintenanceType; break; case TimeoutException: kind = RedisErrorKind.Timeout; @@ -118,6 +120,18 @@ public FaultContext(Exception fault) /// public ConnectionFailureType ConnectionFailureType => _connectionFailureType; + /// + /// The maintenance notification in force when this fault happened, if any. + /// + /// + /// Useful to a circuit breaker: a fault during announced maintenance is expected and transient, and + /// counting it towards a trip that then withdraws a whole server is the opposite of what the notification + /// was for. Left to policy rather than acted on here, since "ignore faults during maintenance" is a + /// judgement about the deployment, not about the protocol. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public Maintenance.MaintenanceNotificationType MaintenanceType { get; } + private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) { // the client never handed it to the socket, so the server cannot have seen it diff --git a/src/StackExchange.Redis/Availability/HealthCheckContext.cs b/src/StackExchange.Redis/Availability/HealthCheckContext.cs index 6d3fd5037..3322ef421 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckContext.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -27,4 +27,16 @@ public readonly struct HealthCheckContext(IServer server, TimeSpan probeTimeout) /// themselves (the caller applies it), but may use it to bound any state they create. /// public TimeSpan ProbeTimeout => probeTimeout; + + /// + /// Gets flags that a probe should include on every command it issues. + /// + /// + /// This marks the traffic as a health check rather than as work a caller is waiting on. It matters because + /// idleness is what decides whether an endpoint can be given up: a probe that looks like caller work makes + /// a server appear busy precisely because we are watching it, and an endpoint that has left the deployment + /// then never gets retired. The built-in probes pass this; a custom probe should too, and passing it + /// changes nothing else about how the command is routed or queued. + /// + public CommandFlags ProbeFlags => Message.ProbeFlag; } diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs index 64b1e882d..3e1491a6c 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs @@ -1,4 +1,4 @@ -using System.Threading.Tasks; +using System.Threading.Tasks; namespace StackExchange.Redis.Availability; @@ -16,7 +16,7 @@ private PingProbe() { } public override async Task CheckHealthAsync(HealthCheckContext context) { - await context.Server.PingAsync(); + await context.Server.PingAsync(context.ProbeFlags); return HealthCheckResult.Healthy; } } diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs index 6e1524b7a..058cff727 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Buffers; using System.Threading.Tasks; @@ -39,10 +39,10 @@ await database.LockTakeAsync( key: key, value: payload, expiry: context.ProbeTimeout, - flags: CommandFlags.FireAndForget).ForAwait(); + flags: CommandFlags.FireAndForget | context.ProbeFlags).ForAwait(); // release from the db if matches (otherwise, we have no clue what happened, so: leave alone) - var success = await database.LockReleaseAsync(key, payload).ForAwait(); + var success = await database.LockReleaseAsync(key, payload, context.ProbeFlags).ForAwait(); return success ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; } finally diff --git a/src/StackExchange.Redis/ClusterConfiguration.cs b/src/StackExchange.Redis/ClusterConfiguration.cs index 2880cf6ac..08fcab15e 100644 --- a/src/StackExchange.Redis/ClusterConfiguration.cs +++ b/src/StackExchange.Redis/ClusterConfiguration.cs @@ -137,21 +137,93 @@ public override int GetHashCode() internal bool Includes(int hashSlot) => hashSlot >= from && hashSlot <= to; - private static bool TryParseInt16(string s, int offset, int count, out short value) + /// + /// Parses one range from within a larger string, so that a comma-separated list can be read without + /// allocating a substring per element. + /// + internal static bool TryParse(string value, int offset, int length, out SlotRange range) { - checked + range = default; + if (length <= 0) return false; + + int dash = -1; + for (int i = 0; i < length; i++) + { + if (value[offset + i] == '-') + { + dash = i; + break; + } + } + + if (dash < 0) { - value = 0; - int tmp = 0; - for (int i = 0; i < count; i++) + if (TryParseInt16(value, offset, length, out var only)) { - char c = s[offset + i]; - if (c < '0' || c > '9') return false; - tmp = (tmp * 10) + (c - '0'); + range = new SlotRange(only, only); + return true; } - value = (short)tmp; + return false; + } + + if (dash != 0 && dash != length - 1 + && TryParseInt16(value, offset, dash, out var from) + && TryParseInt16(value, offset + dash + 1, length - dash - 1, out var to)) + { + // a reversed range is a server bug, not something to normalize silently + if (from > to) return false; + range = new SlotRange(from, to); return true; } + + return false; + } + + /// + /// Parses the comma-and-range slot form used by the maintenance notifications, e.g. + /// 123,456,789-1000. Empty elements are skipped; a malformed element fails the whole list, + /// since a partially-read slot set is worse than none. + /// + internal static bool TryParseList(string? value, out List ranges) + { + ranges = []; + if (string.IsNullOrEmpty(value)) return false; + + int start = 0; + while (start <= value!.Length) + { + var comma = value.IndexOf(',', start); + var length = (comma < 0 ? value.Length : comma) - start; + if (length > 0) + { + if (!TryParse(value, start, length, out var range)) return false; + ranges.Add(range); + } + + if (comma < 0) break; + start = comma + 1; + } + + return ranges.Count != 0; + } + + private static bool TryParseInt16(string s, int offset, int count, out short value) + { + // note this deliberately does not use `checked`: it used to, and an out-of-range slot number + // therefore threw OverflowException from a Try* method - reachable from the public + // SlotRange.TryParse and from CLUSTER NODES parsing, and now also from the maintenance + // notification reader, where throwing on the read loop is far worse than rejecting a value + value = 0; + int tmp = 0; + for (int i = 0; i < count; i++) + { + char c = s[offset + i]; + if (c < '0' || c > '9') return false; + tmp = (tmp * 10) + (c - '0'); + if (tmp > short.MaxValue) return false; // as soon as it cannot fit, stop + } + value = (short)tmp; + return true; } int IComparable.CompareTo(object? obj) => obj is SlotRange sRange ? CompareTo(sRange) : -1; diff --git a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs index 42bb2fcbf..51e2caacf 100644 --- a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs @@ -1,6 +1,8 @@ using System; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Threading.Tasks; +using RESPite; namespace StackExchange.Redis.Configuration { @@ -29,6 +31,9 @@ public class AzureManagedRedisOptionsProvider : DefaultOptionsProvider ".redisenterprise.cache.azure.net", ]; + /// + public override string Name => "amr"; + /// public override bool IsMatch(EndPoint endpoint) { @@ -65,5 +70,18 @@ public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action public override string ConfigurationChannel => ""; // disable on AMR + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// Pre-emptive: AMR does not emit these yet, and support is being added concurrently with this + /// client-side work. is what makes that safe - until + /// the server side ships, the opt-in is refused and the feature stays off, and it then starts working + /// without anybody needing to change a connection string. AMR also already prefers RESP3 here, which + /// the feature requires. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; } } diff --git a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs index c02f8f760..ee577eb59 100644 --- a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs @@ -33,6 +33,9 @@ public class AzureOptionsProvider : DefaultOptionsProvider ".redis.cache.sovcloud-api.fr", }; + /// + public override string Name => "azure"; + /// public override bool IsMatch(EndPoint endpoint) { diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index afcce011f..a278f435f 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using RESPite; namespace StackExchange.Redis.Configuration { @@ -27,6 +29,10 @@ public class DefaultOptionsProvider { new AzureOptionsProvider(), new AzureManagedRedisOptionsProvider(), + new RedisCloudOptionsProvider(), + + // matches nothing, so its position is irrelevant; it is here to be resolvable by name + new RedisEnterpriseOptionsProvider(), }; /// @@ -51,6 +57,68 @@ public static void AddProvider(DefaultOptionsProvider provider) /// public virtual bool IsMatch(EndPoint endpoint) => false; + /// + /// The name this provider can be selected by in a configuration string, as defaults={name}; + /// null (the default) means it cannot be named, and can only be selected in code or by + /// . + /// + /// + /// This exists for the deployments cannot recognize: an on-premise + /// Enterprise cluster has arbitrary DNS, and a hosted one reached through private DNS or a proxy no + /// longer looks like itself. Naming is also what makes a provider expressible in the connection string + /// an application is configured with, rather than only in code it would have to be rebuilt to change. + /// + /// Only providers registered with or built in can be + /// resolved by name - deliberately, since a configuration string that could name an arbitrary type + /// would be a way to have one loaded, and would defeat trimming. + /// + /// + public virtual string? Name => null; + + /// + /// The provider's where it has one, and the type name otherwise. + /// + /// + /// Display only - a provider in a log should read as amr rather than as a namespace-qualified + /// type name. Note that serialization tests directly rather than going through + /// here: this never returns null, and an unnameable provider must not end up in a + /// configuration string. + /// + public override string ToString() => Name ?? base.ToString()!; + + /// + /// Finds a registered provider by its . + /// + internal static bool TryGetByName(string name, [NotNullWhen(true)] out DefaultOptionsProvider? provider) + { + foreach (var candidate in KnownProviders) + { + if (candidate.Name is { } candidateName && string.Equals(candidateName, name, StringComparison.OrdinalIgnoreCase)) + { + provider = candidate; + return true; + } + } + + provider = null; + return false; + } + + /// + /// The names that defaults= accepts, for diagnostics. + /// + internal static string GetKnownNames() + { + var names = new List(); + foreach (var candidate in KnownProviders) + { + if (candidate.Name is { } name && !names.Contains(name)) names.Add(name); + } + + names.Sort(StringComparer.OrdinalIgnoreCase); + return string.Join(", ", names); + } + /// /// Gets a provider for the given endpoints, falling back to if nothing more specific is found. /// @@ -267,6 +335,50 @@ protected virtual string GetDefaultClientName() => /// public virtual RedisProtocol? Protocol => null; + /// + /// Gets whether to ask servers to send maintenance notifications. + /// + /// + /// here on purpose. The notification contract asks + /// clients to default to auto, and that is right for the deployments that emit them - but this + /// library is pointed at every RESP implementation there is, most of which have never heard of the + /// opt-in, and asking them all costs a handshake slot plus an error reply in their logs. Nor is the + /// server the only thing in the path: proxies sit in front of these deployments, and an unrecognized + /// CLIENT subcommand is not guaranteed to be answered as politely as a server would. A provider + /// that recognizes a deployment which supports the feature should override this. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Disabled; + + /// + /// Gets the value command timeouts are relaxed to during an announced disruption; 10 seconds, as the + /// notification contract prescribes. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan MaintenanceRelaxedTimeout => TimeSpan.FromSeconds(10); + + /// + /// Gets the longest a relaxed window may last, or null to derive it as three times the + /// effective . + /// + /// + /// Deriving is the default because the two have to stay coherent: a cap below the timeout it bounds is + /// nonsense, and it is the *configured* relaxed timeout that matters, which this type cannot see. + /// At the prescribed 10 seconds that gives 30 - the contract caps the MOVING budget at 15, so + /// there is ample room for a legitimate window while a stuck one is bounded to half a minute. Return a + /// value here to pin it regardless of the relaxed timeout. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan? MaintenanceRelaxedWindowMax => null; + + /// + /// Gets how long timeouts stay relaxed after a disruption reports completion, or null to derive + /// it as twice the effective + /// (which matches go-redis at the prescribed default). + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan? MaintenancePostEventRelaxedDuration => null; + /// /// Gets whether to enable TCP keep-alive when appropriate (endpoint- and platform-dependent). /// diff --git a/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs b/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs new file mode 100644 index 000000000..fc00c4f9b --- /dev/null +++ b/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs @@ -0,0 +1,83 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Configuration +{ + /// + /// Options provider for Redis Cloud environments. + /// + /// + /// Deliberately *not* a copy of , despite the deployments + /// looking similar: AMR is TLS-only and has a 7.4 floor, and Redis Cloud is neither. See the individual + /// members for what is shared and what is not. + /// + public class RedisCloudOptionsProvider : DefaultOptionsProvider + { + // note the third subsumes the first (EndsWith), so it is kept for the documentation rather than the + // logic - the narrower entry records what the common case actually looks like + private static readonly string[] redisCloudDomains = + [ + ".cloud.redislabs.com", // standard fully-managed endpoints (AWS, GCP, Azure BYOC) + ".cloud.redis.io", // newer routing scheme for managed instances + ".redislabs.com", // older subscriptions, addressed directly + ]; + + /// + public override string Name => "rediscloud"; + + /// + public override bool IsMatch(EndPoint endpoint) + => endpoint is DnsEndPoint dnsEp && IsHostInDomains(dnsEp.Host, redisCloudDomains); + + private static bool IsHostInDomains(string hostName, string[] domains) + { + foreach (var domain in domains) + { + if (hostName.EndsWith(domain, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } + + /// + /// Allow connecting after startup, in the cases where the remote cache isn't ready or is overloaded. + /// + public override bool AbortOnConnectFail => false; + + /// + /// Prefer RESP3, which the deployment supports and which maintenance notifications require. + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; + + /// + /// Disabled: the deployment is proxied, so the OSS configuration-broadcast channel conveys nothing. + /// + public override string ConfigurationChannel => ""; + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// This is the deployment family the feature exists for. + /// rather than because a database that has not been + /// updated yet must keep working: the opt-in is then refused and the feature stays off, rather than the + /// connection being rejected. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; + + // Two things AzureManagedRedisOptionsProvider does that are deliberately *not* repeated here: + // + // - GetDefaultSsl => true. AMR is TLS-only, so assuming TLS there is safe. Redis Cloud enables TLS + // per database and plenty of databases are plaintext, so defaulting it on would fail their connect + // outright - a much worse outcome than not having guessed. + // - DefaultVersion => 7.4. That is AMR's floor. Redis Cloud still offers older versions per database, + // and claiming a version we do not have unlocks commands the server will reject, so the base + // default stands. + } +} diff --git a/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs b/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs new file mode 100644 index 000000000..87634fbf5 --- /dev/null +++ b/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Configuration +{ + /// + /// Options provider for a self-managed Redis Enterprise deployment, selected explicitly. + /// + /// + /// Deliberately matches no endpoint. An on-premise cluster has whatever DNS its operator gave it, so there + /// is nothing to recognize - which is exactly why this provider is nameable: it can be asked for as + /// defaults=enterprise in a configuration string, or assigned to + /// in code. + /// + /// It is also the right choice for a hosted deployment reached somewhere its own provider cannot see it - + /// behind private DNS, a CNAME, or a proxy - where the endpoint no longer looks like what it is. + /// + /// + public class RedisEnterpriseOptionsProvider : DefaultOptionsProvider + { + /// + public override string Name => "enterprise"; + + /// + /// Prefer RESP3, which the deployment supports and which maintenance notifications require. + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; + + /// + /// Disabled: the deployment is proxied, so the OSS configuration-broadcast channel conveys nothing. + /// + public override string ConfigurationChannel => ""; + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// rather than + /// : a cluster that has not been updated yet, or has + /// the feature switched off, must keep working. Choose Enabled explicitly if you would rather a + /// connection be refused than run without advance warning. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; + + // Note: no GetDefaultSsl and no DefaultVersion override. Both are deployment choices here rather than + // properties of the product - TLS is configured per database, and the version is whatever was + // installed - so guessing either would be worse than leaving the library's defaults in place. + } +} diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 12db0bac1..94b95f7c0 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -73,6 +73,49 @@ internal static Proxy ParseProxy(string key, string value) return tmp; } + internal static DefaultOptionsProvider ParseDefaultsProvider(string key, string value) + { + if (!DefaultOptionsProvider.TryGetByName(value, out var provider)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a known defaults provider name; '{value}' is not one of: {DefaultOptionsProvider.GetKnownNames()}."); + } + return provider; + } + + internal static MaintenanceNotificationMode ParseMaintenanceNotifications(string key, string value) + { + if (!Enum.TryParse(value, true, out MaintenanceNotificationMode tmp) || !Enum.IsDefined(typeof(MaintenanceNotificationMode), tmp)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a MaintenanceNotificationMode value; the value '{value}' is not recognised."); + } + return tmp; + } + + /// + /// Parses one of the maintenance durations, which are expressed in seconds - the unit the + /// cross-client contract uses for maintRelaxedTimeout, so a documented value can be pasted + /// between clients. + /// + /// + /// Seconds is the odd one out in this file, where every other timeout is milliseconds, and the + /// mistake is silent in one direction: a caller who assumes milliseconds and writes 30000 would + /// otherwise get an eight-hour relaxed timeout. Hence the upper bound, whose message names the + /// unit - it exists to turn that into a diagnosable error rather than a mystery. + /// + internal static TimeSpan ParseMaintenanceSeconds(string key, string value) + { + const int MaxSeconds = 600; + if (!Format.TryParseInt32(value, out int seconds) || seconds < 0) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a non-negative integer number of seconds; the value '{value}' is not valid."); + } + if (seconds > MaxSeconds) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' is expressed in seconds, and {seconds}s exceeds the maximum of {MaxSeconds}s; note that this option is not in milliseconds."); + } + return TimeSpan.FromSeconds(seconds); + } + internal static SslProtocols ParseSslProtocols(string key, string? value) { // Flags expect commas as separators, but we need to use '|' since commas are already used in the connection string to mean something else @@ -125,6 +168,11 @@ internal const string Tunnel = "tunnel", SetClientLibrary = "setlib", Protocol = "protocol", + Defaults = "defaults", + MaintenanceNotifications = "maintNotifications", + MaintenanceRelaxedTimeout = "maintRelaxedTimeout", + MaintenanceRelaxedWindowMax = "maintRelaxedWindowMax", + MaintenancePostEventRelaxedDuration = "maintPostEventRelaxed", HighIntegrity = "highIntegrity", TcpKeepAlive = "tcpKeepAlive"; @@ -162,6 +210,11 @@ internal const string Tunnel, SetClientLibrary, Protocol, + Defaults, + MaintenanceNotifications, + MaintenanceRelaxedTimeout, + MaintenanceRelaxedWindowMax, + MaintenancePostEventRelaxedDuration, HighIntegrity, TcpKeepAlive, }.ToDictionary(x => x, StringComparer.OrdinalIgnoreCase); @@ -217,6 +270,11 @@ private enum OptionFlags : ulong SslProtocolsHasValue = 1UL << 32, ProtocolHasValue = 1UL << 33, AllowSimulateConnectionFailure = 1UL << 34, + MaintenanceNotificationsHasValue = 1UL << 35, + DefaultsHasValue = 1UL << 36, + MaintenanceRelaxedTimeoutHasValue = 1UL << 37, + MaintenanceRelaxedWindowMaxHasValue = 1UL << 38, + MaintenancePostEventRelaxedDurationHasValue = 1UL << 39, } private OptionFlags optionFlags; @@ -242,6 +300,8 @@ private enum OptionFlags : ulong private SslProtocols sslProtocols; private RedisProtocol _protocol; + private MaintenanceNotificationMode _maintenanceNotifications; + private TimeSpan _maintenanceRelaxedTimeout, _maintenanceRelaxedWindowMax, _maintenancePostEventRelaxedDuration; private bool HasValue(OptionFlags hasValue) => (optionFlags & hasValue) != 0; @@ -314,7 +374,15 @@ private void SetWithValue(OptionFlags hasValue, ref T storage, T value) where public DefaultOptionsProvider Defaults { get => defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); - set => defaultOptions = value; + set + { + defaultOptions = value; + + // the getter memoizes an *inferred* provider into the same field, so a flag is the only way to + // tell "the caller chose this" from "we worked it out from the endpoints" - and only the former + // may be written back out to a configuration string + optionFlags |= OptionFlags.DefaultsHasValue; + } } /// @@ -1004,6 +1072,10 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow Tunnel = Tunnel, LibraryName = LibraryName, _protocol = _protocol, + _maintenanceNotifications = _maintenanceNotifications, + _maintenanceRelaxedTimeout = _maintenanceRelaxedTimeout, + _maintenanceRelaxedWindowMax = _maintenanceRelaxedWindowMax, + _maintenancePostEventRelaxedDuration = _maintenancePostEventRelaxedDuration, heartbeatInterval = heartbeatInterval, WriteMode = WriteMode, CircuitBreaker = CircuitBreaker, @@ -1097,6 +1169,13 @@ public string ToString(bool includePassword) Append(sb, OptionKeys.SetClientLibrary, OptionFlags.SetClientLibraryHasValue, OptionFlags.SetClientLibraryValue); Append(sb, OptionKeys.HighIntegrity, OptionFlags.HighIntegrityHasValue, OptionFlags.HighIntegrityValue); if (HasValue(OptionFlags.ProtocolHasValue)) Append(sb, OptionKeys.Protocol, FormatProtocol(_protocol)); + // only when the caller set it *and* it can be named: an inferred provider must not be baked into + // the string, or re-parsing would pin a choice that was only ever a guess from the endpoints + if (HasValue(OptionFlags.DefaultsHasValue) && defaultOptions?.Name is { } defaultsName) Append(sb, OptionKeys.Defaults, defaultsName); + if (HasValue(OptionFlags.MaintenanceNotificationsHasValue)) Append(sb, OptionKeys.MaintenanceNotifications, _maintenanceNotifications.ToString()); + if (HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedTimeout, FormatMaintenanceSeconds(_maintenanceRelaxedTimeout)); + if (HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedWindowMax, FormatMaintenanceSeconds(_maintenanceRelaxedWindowMax)); + if (HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue)) Append(sb, OptionKeys.MaintenancePostEventRelaxedDuration, FormatMaintenanceSeconds(_maintenancePostEventRelaxedDuration)); Append(sb, OptionKeys.TcpKeepAlive, OptionFlags.TcpKeepAliveHasValue, OptionFlags.TcpKeepAliveValue); if (Tunnel is { IsInbuilt: true } tunnel) { @@ -1212,6 +1291,8 @@ private void Clear() #endif Tunnel = null; _protocol = default; + _maintenanceNotifications = default; + _maintenanceRelaxedTimeout = _maintenanceRelaxedWindowMax = _maintenancePostEventRelaxedDuration = default; WriteMode = default; CircuitBreaker = null; RetryPolicy = null; @@ -1368,6 +1449,21 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) case OptionKeys.Protocol: SetWithValue(OptionFlags.ProtocolHasValue, ref _protocol, OptionKeys.ParseRedisProtocol(key, value)); break; + case OptionKeys.Defaults: + Defaults = OptionKeys.ParseDefaultsProvider(key, value); + break; + case OptionKeys.MaintenanceNotifications: + SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, OptionKeys.ParseMaintenanceNotifications(key, value)); + break; + case OptionKeys.MaintenanceRelaxedTimeout: + SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; + case OptionKeys.MaintenanceRelaxedWindowMax: + SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; + case OptionKeys.MaintenancePostEventRelaxedDuration: + SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; // Deprecated options we ignore... case OptionKeys.HighPrioritySocketThreads: case OptionKeys.PreserveAsyncOrder: @@ -1420,6 +1516,86 @@ public RedisProtocol? Protocol set => Set(OptionFlags.ProtocolHasValue, ref _protocol, value); } + /// + /// Whether to ask servers to send maintenance notifications; note that + /// means required and rejects connections + /// that cannot deliver them - is the best-effort + /// mode. + /// + /// + /// Requires RESP3, and only Redis Enterprise and Redis Cloud emit them - so the default is + /// rather than spending an extra handshake command + /// asking every server in existence a question almost none of them understand. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationMode MaintenanceNotifications + { + get => HasValue(OptionFlags.MaintenanceNotificationsHasValue) ? _maintenanceNotifications : Defaults.MaintenanceNotifications; + set => SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, value); + } + + /// + /// The value command timeouts are relaxed to while a server has announced a disruption. + /// + /// + /// This is a floor, never a reduction: the effective timeout inside a window is + /// max(configured, this), so a caller with a generous timeout keeps it. Expressed in seconds in + /// a configuration string (maintRelaxedTimeout=10), matching the cross-client contract - note + /// that this is unlike every other timeout here, which are milliseconds. Only command timeouts are + /// relaxed; keep-alive, the heartbeat and connection-failure detection are deliberately untouched, so + /// a server that dies mid-maintenance is still noticed on the usual schedule. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenanceRelaxedTimeout + { + get => HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue) ? _maintenanceRelaxedTimeout : Defaults.MaintenanceRelaxedTimeout; + set => SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, value); + } + + /// + /// The longest a relaxed window may last, however long the server said the disruption would take. + /// + /// + /// A backstop for a closing notification that never arrives, and our invention - the + /// notification contract names no upper bound. A window that never closes is worse than one that + /// closes early, since relaxation delays the point at which a genuinely slow server surfaces as a + /// timeout. Expressed in seconds in a configuration string. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenanceRelaxedWindowMax + { + get => HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue) + ? _maintenanceRelaxedWindowMax + : Defaults.MaintenanceRelaxedWindowMax ?? Multiply(MaintenanceRelaxedTimeout, 3); + set => SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + } + + /// + /// How long to keep timeouts relaxed after a disruption reports that it has finished. + /// + /// + /// Also our invention. A closing notification means the server-side operation completed, not + /// that the server is back to normal latency - and the moment after it completes is precisely when + /// every other client that received the same notification re-engages, so the load spike arrives + /// slightly after the all-clear. Does not apply when a window ended by hitting + /// : in that case nothing told us the event finished, and + /// extending past the backstop would defeat it. Expressed in seconds in a configuration string. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenancePostEventRelaxedDuration + { + get => HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue) + ? _maintenancePostEventRelaxedDuration + : Defaults.MaintenancePostEventRelaxedDuration ?? Multiply(MaintenanceRelaxedTimeout, 2); + set => SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + } + + // TimeSpan * int is netstandard2.1+, so this is ticks arithmetic to keep the down-level TFMs building + private static TimeSpan Multiply(TimeSpan value, int factor) => TimeSpan.FromTicks(value.Ticks * factor); + + private static string FormatMaintenanceSeconds(TimeSpan value) + => ((int)value.TotalSeconds).ToString(System.Globalization.CultureInfo.InvariantCulture); + internal BufferedStreamWriter.WriteMode WriteMode { get; set; } /// diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs index 0a8b95be5..fd5bd7c7f 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs @@ -88,6 +88,69 @@ private void OnEndpointChanged(EndPoint endpoint, EventHandler public event EventHandler? ServerMaintenanceEvent; + + /// + /// How host names are resolved during a maintenance handoff. + /// + /// + /// A seam rather than a call to directly, because no in-process fake can move a + /// DNS record: the handoff's whole decision turns on the answer *changing*, and the only way to test that + /// deterministically is to supply the answers. Defaults to real DNS. + /// + internal Func> AddressResolver { get; set; } + = Maintenance.AdvertisedAddressProbe.DefaultResolveAsync; + + // recently-raised (type, sequence) pairs, so one logical event raises one event however many nodes told + // us. Small and fixed: the copies arrive within milliseconds of each other, so a handful of slots covers + // any realistic proxy count even with other notifications interleaved. + // A named struct rather than a tuple: this assembly must not reference System.ValueTuple, which breaks + // .NET Framework consumers - see SanityChecks.ValueTupleNotReferenced. + private readonly struct RaisedMaintenanceEvent(Maintenance.MaintenanceNotificationType type, long sequence) + { + public readonly Maintenance.MaintenanceNotificationType Type = type; + public readonly long Sequence = sequence; + } + + private readonly RaisedMaintenanceEvent[] _raisedMaintenanceEvents = new RaisedMaintenanceEvent[8]; + private int _raisedMaintenanceEventIndex; + + /// + /// Whether this is the first time we have been told about a given maintenance event, across every + /// connection. + /// + /// + /// Every node broadcasts a given event, and all of them carry the same sequence number - observed on + /// Enterprise 8.6.2, where the id identifies the event rather than the delivery. So without this, a + /// deployment fronted by three proxies raises three events for one migration and every consumer has to + /// dedupe them. + /// + /// Matched on equality rather than "less than or equal", deliberately: a lagging node reporting an + /// *earlier* event we have not seen yet is a distinct event and must still be raised. Only an exact repeat + /// of something already raised is a duplicate. + /// + /// + /// Eviction is the only expiry - an entry falls out once eight further notifications have been recorded, so + /// nothing has to be purged on a timer and the state cannot grow. A duplicate arriving after its entry has + /// been evicted would be raised a second time, which is the right way round to be wrong: the copies arrive + /// within milliseconds of each other, so that takes a straggler behind eight intervening events. + /// + /// + internal bool TryClaimMaintenanceEvent(Maintenance.MaintenanceNotificationType type, long? sequence) + { + if (sequence is not { } seq) return true; // no id to match on; better a duplicate than a silence + + lock (_raisedMaintenanceEvents) + { + foreach (var entry in _raisedMaintenanceEvents) + { + if (entry.Type == type && entry.Sequence == seq) return false; + } + + _raisedMaintenanceEvents[_raisedMaintenanceEventIndex] = new RaisedMaintenanceEvent(type, seq); + _raisedMaintenanceEventIndex = (_raisedMaintenanceEventIndex + 1) % _raisedMaintenanceEvents.Length; + return true; + } + } internal void OnServerMaintenanceEvent(ServerMaintenanceEvent e) => ServerMaintenanceEvent?.Invoke(this, e); diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 903941c1f..d4182cf41 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2492,7 +2492,25 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u throw GetException(result, message, server); } - if (Monitor.Wait(source, TimeoutMilliseconds)) + // Unlike the async sweeps, which re-read the effective timeout on every heartbeat, a + // sync caller commits to a duration when it parks - so if maintenance relaxation + // begins while we are waiting, we have to notice by re-waiting rather than failing at + // the original deadline. Without this a sync caller in flight when a MIGRATING + // arrives times out at the strict timeout while its async neighbour is relaxed. + var watch = ValueStopwatch.StartNew(); + bool completed = Monitor.Wait(source, server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds); + while (!completed) + { + var elapsed = watch.ElapsedMilliseconds; + var revised = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds; + + // also the path back: if a window closed while we waited, revised drops to the + // configured value and we stop + if (revised <= elapsed) break; + completed = Monitor.Wait(source, revised - elapsed); + } + + if (completed) { Trace("Timely response to " + message); } diff --git a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs index 9f0df5ba0..3fd7847b3 100644 --- a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs +++ b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs @@ -68,5 +68,22 @@ public enum ConnectionFailureType /// [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] CircuitBreaker, + + /// + /// The connection was replaced deliberately, to move off an endpoint the server announced it is + /// retiring. + /// + /// + /// Not a fault: the connection was working, and we chose to replace it while a replacement address was + /// known rather than wait to be disconnected. Reported here because the alternative is worse - the + /// replacement raises , so without this a consumer + /// tracking connection state sees a restore with no matching failure, and no reason for the churn. + /// + /// Consumers alerting on should filter this out: + /// it is expected during planned maintenance and says nothing is wrong. is + /// the precedent for a deliberate client action being reported this way. + /// + /// + MaintenanceHandoff, } } diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index aa9bf7001..c2977de0e 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -319,14 +319,19 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; + // if the server had announced a disruption, say so on the fault: "timeout" and "timeout during an + // announced failover" call for very different reactions from whoever reads the log + var maintenanceType = server?.ActiveMaintenanceType ?? Maintenance.MaintenanceNotificationType.None; Exception ex = logConnectionException && lastConnectionException is not null ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, + MaintenanceType = maintenanceType, } : new RedisTimeoutException(msgFlags, sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, + MaintenanceType = maintenanceType, }; CopyDataToException(data, ex); diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 03c62ed95..7febd6dde 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Text; using System.Threading.Tasks; @@ -765,4 +765,10 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) EventId = 110, Message = "{BridgeName}: Transport connected (encrypted: {IsEncrypted})")] internal static partial void LogInformationTransportConnected(this ILogger logger, string bridgeName, bool isEncrypted); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 116, + Message = "{Server}: Requesting maintenance notifications ({Mode})")] + internal static partial void LogInformationRequestingMaintenanceNotifications(this ILogger logger, ServerEndPointLogValue server, MaintenanceNotificationMode mode); } diff --git a/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs b/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs new file mode 100644 index 000000000..6170e4dc1 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs @@ -0,0 +1,227 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Maintenance; + +/// +/// Asks DNS the two questions a handoff needs: what replaces the address we are leaving, and are we still +/// advertised at all. +/// +/// +/// A poll rather than a lookup, and that is the whole point. Measured on Redis Enterprise (2026-08-28), with +/// times relative to the notification: the server-side endpoint moves at +8.6s, DNS follows at +9.7s and +4.4s +/// across two runs, and the sockets close at +18.4s and +16.6s - against a declared 15s grace and a 5s record +/// TTL. So resolving immediately returns the address being retired, in every run observed, and a +/// client that treats the first answer as authoritative hands off to the node it was told to leave. +/// +/// DNS is not guaranteed to win the race. On a second cluster the same notification with the same 15s +/// grace saw the socket close at +15.7s and DNS update at +18.7s - three seconds after the close. So +/// there are three outcomes here, not two, and the third is a normal one: the record may still be stale when +/// the window runs out, and the only thing left is to reconnect after the close and resolve then. Do not +/// "simplify" the null return away. +/// +/// +/// There is no way to observe the intermediate state from outside: the endpoint has moved server-side well +/// before DNS reflects it, and nothing tells a client which of those has happened. Probing until the answer +/// changes is the only mechanism available, and the short TTL is what makes it work - several attempts fit +/// inside the window. +/// +/// +/// The rule is "take any address that is not the one being retired". Note it is deliberately *not* "prefer an +/// address that has newly appeared", even though a MOVING implies one exists - it is emitted when the +/// connection's own proxy *leaves* the endpoint's address set **and** the set gains a member (established +/// across thirteen observations: a pure reduction takes the proxy away without announcing anything, and a pure +/// widening adds addresses while leaving the connection's proxy in place, which is equally silent). A live +/// sibling is at least as good a target as a newly joined node and is available *now*, whereas the new node +/// only becomes visible when the record updates - which can be after the socket has already closed. Preferring +/// the newcomer would mean waiting for it, and waiting is the thing to avoid. The one case where it would pay +/// is a rolling operation, where the sibling we step to may take its own turn later; that costs one further +/// handoff, bounded at one per connection per operation, which is cheaper than a lost window. +/// +/// +/// This rule is also doing more work than it looks. A Redis Cloud hostname usually carries several A records - measured 2026-08-28: 2 for +/// all-nodes, 3 for all-master-shards, 1 for single, all on a 5s TTL - and the count +/// follows actual proxy *placement* rather than the policy name, so a multi-proxy database whose shards happen +/// to share a node still resolves to one address. With several records the first resolution already names a +/// live sibling proxy, so this returns immediately and steps sideways rather than waiting: any proxy of the +/// same database serves the same data, so a sibling now beats the replacement in nine seconds. The poll only +/// engages when the record names nothing but the address being retired - the single-address case, which is +/// exactly where waiting is the only option available. +/// +/// +/// Deliberately free of jitter, clocks-by-configuration and connection state, so that it can be tested +/// exhaustively without a server: the caller supplies the resolver, the interval and the budget. Jitter +/// belongs at the call site, where the existing refresh jitter lives. +/// +/// +internal static class AdvertisedAddressProbe +{ + /// + /// Ordinary DNS, which is what everything but a test uses. + /// + internal static Task DefaultResolveAsync(string host, CancellationToken cancellationToken) => +#if NET + Dns.GetHostAddressesAsync(host, cancellationToken); +#else + Dns.GetHostAddressesAsync(host); +#endif + + /// + /// Polls DNS until it stops naming , or until the window runs out. + /// + /// + /// The replacement endpoint, or null if the window expired without DNS moving - a measured outcome + /// rather than a failure (see the type remarks: DNS has been seen updating three seconds after the socket + /// closed). The caller should then do nothing proactive: the server closes the socket, the reconnect that + /// follows re-resolves anyway because the endpoint is a hostname, and the relaxed timeout window covers + /// the gap. Guessing an address here would be strictly worse. + /// + internal static async Task ProbeAsync( + DnsEndPoint endpoint, + IPAddress retiring, + TimeSpan window, + TimeSpan pollInterval, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (retiring is null) throw new ArgumentNullException(nameof(retiring)); + if (resolve is null) throw new ArgumentNullException(nameof(resolve)); + + // A non-positive window is not an error: a notification can arrive with nothing left of its budget + // (the shard notifications legitimately carry zero or negative times), and "act now" means one attempt + // rather than none. + var deadline = Environment.TickCount + (int)Math.Max(window.TotalMilliseconds, 0); + int attempt = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + attempt++; + + IPAddress[]? addresses = null; + try + { + addresses = await resolve(endpoint.Host, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // a DNS blip mid-handoff is exactly when we can least afford to give up; keep trying until + // the window says otherwise + log?.Invoke($"MOVING: resolve attempt {attempt} for {endpoint.Host} failed: {ex.Message}"); + } + + if (addresses is not null) + { + IPAddress? candidate = null; + bool retiringStillAdvertised = false; + foreach (var address in addresses) + { + if (address.Equals(retiring)) retiringStillAdvertised = true; + else candidate ??= address; + } + + if (candidate is not null) + { + // Two operationally different outcomes, worth distinguishing in a log somebody reads while + // debugging a handoff: we either stepped sideways to a proxy that was already there, or the + // record itself has moved on to the replacement. + log?.Invoke(retiringStillAdvertised + ? $"MOVING: {endpoint.Host} still advertises {retiring}; moving to sibling {candidate} (attempt {attempt})" + : $"MOVING: {endpoint.Host} now resolves to {candidate} after {attempt} attempt(s)"); + return new IPEndPoint(candidate, endpoint.Port); + } + + log?.Invoke($"MOVING: {endpoint.Host} still resolves only to {retiring} (attempt {attempt}); not yet updated"); + } + + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining <= 0) + { + log?.Invoke($"MOVING: {endpoint.Host} never stopped resolving to {retiring} within the window"); + return null; + } + + // never sleep past the deadline: the last attempt should land inside the window, not after it + var delay = (int)Math.Min(pollInterval.TotalMilliseconds, remaining); + if (delay > 0) await Task.Delay(delay, cancellationToken).ForAwait(); + } + } + + /// + /// Whether the address we are connected on is still one of the addresses the hostname advertises. + /// + /// + /// true if still advertised, false if the record no longer names it, and null if DNS + /// could not be asked - which is *not* the same as "no": a resolution failure is no reason to give up a + /// working connection. + /// + /// + /// The trigger that needs no notification, and the reason it matters is a measured gap. On a multi-proxy + /// database, taking a node out for maintenance announced only the data-movement pair - MIGRATING at + /// +4.3s and MIGRATED at +16.6s, to every proxy - and then dropped the victim from DNS at +21.4s and + /// closed its socket *silently* at +34.7s, while sibling connections stayed up past +90s. No + /// MOVING was ever sent. + /// + /// So for thirteen seconds the condition was plainly visible to anybody who asked - our address is no + /// longer advertised - and the client's only other signal was the socket dying with no explanation. Asking + /// this question when a MIGRATED arrives converts that into a controlled handoff. Note + /// MIGRATED is also the notification the server *retains* and replays on connect, so a client that + /// arrives mid-operation gets the same prompt. + /// + /// + /// The same condition is what a support case turned on: a client kept dialling endpoints that no longer + /// existed for 37 hours, because nothing it could observe told it to stop. Two unrelated failures, one + /// detection rule. + /// + /// + internal static async Task IsStillAdvertisedAsync( + DnsEndPoint endpoint, + IPAddress current, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (current is null) throw new ArgumentNullException(nameof(current)); + if (resolve is null) throw new ArgumentNullException(nameof(resolve)); + + IPAddress[] addresses; + try + { + addresses = await resolve(endpoint.Host, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + log?.Invoke($"{endpoint.Host}: cannot tell whether {current} is still advertised: {ex.Message}"); + return null; + } + + // an empty answer is "cannot tell" rather than "no": a record that momentarily resolves to nothing is + // a DNS problem, not an instruction to abandon a connection that is working + if (addresses is null || addresses.Length == 0) + { + log?.Invoke($"{endpoint.Host}: resolved to nothing; treating {current} as still advertised"); + return null; + } + + foreach (var address in addresses) + { + if (address.Equals(current)) return true; + } + + log?.Invoke($"{endpoint.Host}: no longer advertises {current}; it is being taken out of service"); + return false; + } +} diff --git a/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs b/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs new file mode 100644 index 000000000..7a9b7d950 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// One entry from a cluster slot-migration notification: some slots have moved, or are moving, from one node +/// to another. +/// +/// +/// A single SMIGRATED describes several of these at once, and not necessarily any involving the node +/// that sent it - every node in the cluster reports the same movements, so the sender is not implicitly the +/// source. +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public readonly struct ClusterSlotMigration +{ + internal ClusterSlotMigration(EndPoint? source, EndPoint? target, IReadOnlyList slots, string? raw) + { + Source = source; + Target = target; + Slots = slots; + RawSlots = raw; + } + + /// + /// The node the slots are moving from, or null if the server named it in a form that cannot be + /// dialled. + /// + public EndPoint? Source { get; } + + /// + /// The node the slots are moving to, or null if the server named it in a form that cannot be + /// dialled. + /// + public EndPoint? Target { get; } + + /// + /// The slots that are moving. + /// + public IReadOnlyList Slots { get; } + + /// + /// The slots exactly as the server expressed them, for the cases the parsed form does not survive - a + /// malformed list leaves empty and this populated. + /// + public string? RawSlots { get; } + + /// + public override string ToString() => $"{Format.ToString(Source)} -> {Format.ToString(Target)}: {RawSlots}"; +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs new file mode 100644 index 000000000..57959cdce --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs @@ -0,0 +1,47 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +/// +/// Carries maintenance context on the faults a disruption can cause, so that a timeout during an announced +/// migration is distinguishable from an ordinary one. +/// +/// +/// This follows the established pattern on these types - Commandstatus and Flags on +/// , FailureType on - rather +/// than introducing an exception type nobody is catching yet. Both properties are named for the role they +/// play, not for the type, matching FailureType. +/// +public sealed partial class RedisTimeoutException +{ + /// + /// The maintenance notification in force when this timed out, or + /// if the server had not announced anything. + /// + /// + /// A value here says the server told us to expect disruption, and that the timeout you are looking at + /// happened inside that window - including the tail after the disruption reported completion. It does not + /// promise that the maintenance *caused* the timeout, only that the two coincided; that is still the most + /// useful thing to know when reading a log after the fact. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationType MaintenanceType { get; internal init; } +} + +/// +public sealed partial class RedisConnectionException +{ + /// + /// The maintenance notification in force when this connection faulted, or + /// if the server had not announced anything. + /// + /// + /// Present on this type as well as on because a handoff that misses + /// its deadline surfaces either way round: as a timeout when the command was already in flight, and as a + /// connection fault when the endpoint went away underneath it. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationType MaintenanceType { get; internal init; } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs new file mode 100644 index 000000000..38df5c310 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs @@ -0,0 +1,133 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Maintenance; + +/// +/// What to do about a MOVING notification. +/// +internal enum HandoffAction +{ + /// Nothing useful is available; let the server close the socket and reconnect then. + None, + + /// Drop our connections so they re-establish against the replacement address. + Recycle, + + /// The replacement is a different endpoint; the topology has to be re-read to find it. + Reconfigure, +} + +/// +/// The outcome of deciding how to hand off, and why - the reason is logged either way. +/// +internal readonly struct HandoffDecision(HandoffAction action, EndPoint? target, string reason) +{ + public HandoffAction Action { get; } = action; + + public EndPoint? Target { get; } = target; + + public string Reason { get; } = reason; + + public override string ToString() => Target is null ? $"{Action}: {Reason}" : $"{Action} -> {Target}: {Reason}"; +} + +/// +/// Decides what a MOVING notification means for this connection, and where to go. +/// +/// +/// Separated from the acting so that it can be tested exhaustively without a server: the caller supplies the +/// endpoint, the address it is currently on, the announced window and a resolver, and gets back an action. +/// +/// The dispatch turns on the *form* of the endpoint, which is the thing that decides whether a handoff is even +/// possible - and the answer corrects an earlier assumption that MOVING should reuse the endpoint +/// retirement path: +/// +/// +/// +/// A hostname endpoint with no named successor - the case every observed MOVING has been - keeps its +/// : only the address behind the name moves. Retiring the endpoint would delete our +/// only way of reaching the deployment. So the action is to recycle the connections once DNS has moved, which +/// re-resolves them. +/// +/// +/// An address endpoint with a named successor is genuinely a different endpoint, so the topology has to be +/// re-read. Never observed: eleven routes have all carried an explicit null, including cases where the server +/// had already chosen the replacement node. Treat this branch as code that must exist rather than code that is +/// exercised. +/// +/// +/// An address endpoint with no successor has nothing to re-resolve and nowhere named to go. Doing nothing is +/// correct: the socket closes, the reconnect happens, and the relaxed window covers it. +/// +/// +/// +internal static class MaintenanceHandoff +{ + internal static async Task DecideAsync( + EndPoint endpoint, + EndPoint? successor, + IPAddress? currentAddress, + TimeSpan window, + TimeSpan pollInterval, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (successor is not null) + { + // Nothing is asserted about *which* endpoint: a successor may be an address we have never seen, so + // finding it is a topology question rather than a DNS one. + return new HandoffDecision(HandoffAction.Reconfigure, successor, "the server named a replacement endpoint"); + } + + if (endpoint is not DnsEndPoint dns) + { + return new HandoffDecision( + HandoffAction.None, + null, + $"{Format.ToString(endpoint)} is an address, not a name, and no replacement was named: nothing to re-resolve"); + } + + if (currentAddress is null) + { + // Without knowing where we are, "has it moved" is unanswerable - and guessing would mean recycling + // onto whatever DNS says right now, which for the first several seconds is the address being retired. + return new HandoffDecision(HandoffAction.None, null, "the address of the current connection is unknown"); + } + + var replacement = await AdvertisedAddressProbe.ProbeAsync( + dns, currentAddress, window, pollInterval, resolve, log, cancellationToken).ForAwait(); + + return replacement is null + ? new HandoffDecision( + HandoffAction.None, + null, + $"{dns.Host} still resolved only to {currentAddress} when the window expired") + : new HandoffDecision( + HandoffAction.Recycle, + replacement, + $"{dns.Host} now resolves to {replacement}"); + } + + /// + /// How long to wait before probing, so a fleet does not resolve in lockstep. + /// + /// + /// A *fraction* of the announced window rather than a fixed delay, which is the difference from the refresh + /// jitter elsewhere. Windows are not always generous - the shard notifications have been measured announcing + /// two seconds - so a flat one-second jitter could spend half of one, and on a short window the right amount + /// of jitter is almost none. Capped as well as scaled, because a long window does not justify a long wait + /// when DNS has been seen moving after four seconds. + /// + internal static TimeSpan GetJitter(TimeSpan window, Random random) + { + if (window <= TimeSpan.Zero) return TimeSpan.Zero; + + var tenth = window.TotalMilliseconds / 10; + var capped = Math.Min(tenth, 1000); + return TimeSpan.FromMilliseconds(random.Next(0, (int)Math.Max(capped, 1))); + } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs b/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs new file mode 100644 index 000000000..ae66e4ce2 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs @@ -0,0 +1,58 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// The kind of a server-native maintenance notification. +/// +/// +/// Two families share this enum: the Enterprise proxy notifications ( through +/// ) and the OSS cluster ones (, +/// ). Unrecognized types are dropped rather than surfaced, so no member here means +/// "something we didn't understand". +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceNotificationType +{ + /// + /// Not a maintenance notification; the default, for contexts where this is simply not applicable. + /// + None = 0, + + /// + /// This endpoint is being replaced; the notification names its successor, or names nothing at all if the + /// server has no address to offer. + /// + Moving, + + /// + /// A shard is migrating away from this node. Expect latency; expect a after it. + /// + Migrating, + + /// + /// A migration announced by has completed. + /// + Migrated, + + /// + /// This node is failing over. Expect latency; expect a after it. + /// + FailingOver, + + /// + /// A failover announced by has completed. + /// + FailedOver, + + /// + /// Slots are migrating (the OSS cluster family). + /// + SlotMigrating, + + /// + /// Slots announced by have migrated. + /// + SlotMigrated, +} diff --git a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs new file mode 100644 index 000000000..550d1e1de --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// A server-native maintenance notification, received as a RESP3 push frame on the connection that carries +/// commands. One class with a discriminator rather than a type per +/// notification: the payloads are near-identical, and it keeps the handling in one place. +/// +/// +/// Observation only at present: receiving one of these raises +/// and does nothing else. Acting on them - relaxing +/// timeouts, then moving off a doomed endpoint - is deliberately separate work, so a consumer can watch what +/// its servers are announcing before any behaviour depends on it. +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public sealed class PushMaintenanceEvent : ServerMaintenanceEvent +{ + internal PushMaintenanceEvent( + MaintenanceNotificationType notificationType, + long sequenceId, + EndPoint? endPoint, + TimeSpan? time, + EndPoint? newEndPoint, + string? payload, + string rawMessage, + IReadOnlyList? slotMigrations = null) + { + NotificationType = notificationType; + SequenceId = sequenceId; + EndPoint = endPoint; + Time = time; + NewEndPoint = newEndPoint; + Payload = payload; + RawMessage = rawMessage; + SlotMigrations = slotMigrations ?? []; + if (time is { } value && value > TimeSpan.Zero) + { + StartTimeUtc = ReceivedTimeUtc + value; + } + } + + /// + /// Which notification this is. + /// + public MaintenanceNotificationType NotificationType { get; } + + /// + /// The sequence number the server attached to this notification. + /// + /// + /// No specification defines these, but observation does: on Enterprise 8.6.2 they are monotonic per + /// database, start at zero on a fresh one, are shared *across* notification types (a + /// at 16 followed by its + /// at 17), and carry the same value on every node + /// that broadcasts a given event - so they identify the event rather than the connection that delivered + /// it. That makes them genuinely useful for spotting a replay. + /// + /// Still treat cross-deployment use as heuristic: this is one build of one product, and nothing obliges a + /// different implementation to behave the same way. + /// + /// + public long SequenceId { get; } + + /// + /// The server that sent the notification. + /// + /// + /// More precisely: whichever node told us first. Every node broadcasts a given event, so a three-proxy + /// deployment delivers one migration three times, on three connections, with the same + /// . All three are acted on internally - the timeout relaxation is per-server, so + /// each connection genuinely has to see it - but the event is raised once, for the first arrival, and the + /// rest are dropped. Do not read this as "the node being maintained": for the cluster notifications it is + /// usually a bystander reporting someone else's movements (see ). + /// + public EndPoint? EndPoint { get; } + + /// + /// How long the announced event is expected to take, or how much of it remains. + /// + /// + /// The meaning is per-notification: for it is the budget + /// for completing the move, and for the others it is the remaining duration of the announced disruption. + /// It can legitimately be zero or negative - a connection that arrives mid-window is told what is left of + /// it - which means "act now" rather than being an error. null where the notification carries no + /// time at all. + /// + public TimeSpan? Time { get; } + + /// + /// For , the endpoint this one is being replaced by. + /// + /// + /// null is a documented outcome, not just a parse failure: a server with no address to offer sends + /// an explicit null, and it also does so when it cannot honour the endpoint type that was requested. The + /// intended handling in that case is to reconnect using the endpoint already configured, rather than to + /// treat the notification as invalid. See for what actually arrived. + /// + public EndPoint? NewEndPoint { get; } + + /// + /// The final element of the notification, as received: the affected shard ids, the affected slots, or the + /// endpoint of a . + /// + /// + /// Deliberately opaque. Nothing a client is asked to *do* depends on which shards are involved, so this is + /// carried through for diagnostics rather than parsed into a model that the contract does not pin down. + /// + public string? Payload { get; } + + /// + /// For the cluster notifications, the slot movements described; empty for everything else. + /// + /// + /// A notification carries several of these, and the node that sent it is not necessarily the source of + /// any of them - every node reports the same movements. + /// + public IReadOnlyList SlotMigrations { get; } + + /// + public override string? ToString() => RawMessage; +} diff --git a/src/StackExchange.Redis/MaintenanceNotificationMode.cs b/src/StackExchange.Redis/MaintenanceNotificationMode.cs new file mode 100644 index 000000000..fc7c15623 --- /dev/null +++ b/src/StackExchange.Redis/MaintenanceNotificationMode.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis; + +/// +/// Whether to ask a server to send maintenance notifications - advance warning of migrations, failovers and +/// endpoint moves. +/// +/// +/// The three modes, and their names, are prescribed cross-client so that configuration and support tickets +/// line up between clients. Only Redis Enterprise and Redis Cloud emit these notifications; OSS Redis, Valkey +/// and Garnet do not recognize the opt-in at all, which is why exists and why the default +/// is rather than asking every server in existence. +/// +/// Note that means "required", not "on": it rejects any connection that cannot +/// deliver notifications. is the mode that turns the feature on where available and +/// stays out of the way otherwise, and is what most callers want. The names are the cross-client ones, and +/// this is the one place they are easy to misread. +/// +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceNotificationMode +{ + /// + /// Never ask; no notification is requested and none will arrive. + /// + Disabled = 0, + + /// + /// Requires them, and REJECTS THE CONNECTION if they are unavailable - including against any + /// server that does not support them, and on any RESP2 connection. Only for a deployment known to + /// support them; use to ask without that risk. + /// + /// + /// A server that refuses fails the connection, and so does a server that answers HELLO 3 as RESP2, + /// since nothing can be delivered on a RESP2 connection. So does a configuration that could never ask in + /// the first place - Protocol = Resp2, or HELLO unavailable - because requiring a + /// RESP3-only feature over RESP2 is a contradiction, and honouring half of it silently is the outcome + /// this mode exists to prevent. + /// + Enabled, + + /// + /// Ask, and carry on if the server refuses or the connection ends up RESP2 - the feature is then simply + /// off for that server, and the connection is never rejected over it. Safe against a mixture of servers, + /// or one whose support you don't know. + /// + Auto, +} diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 837651106..c6aceb60b 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -62,7 +62,10 @@ internal const CommandFlags NoFlushFlag = (CommandFlags)1024, // "server specific" (bit 18): tied to a specific endpoint, never retry elsewhere. Not (yet) a // public CommandFlags member - see the note on the hidden bit-18 value in CommandFlags.cs. - CommandServerSpecific = (CommandFlags)(1 << 18); + CommandServerSpecific = (CommandFlags)(1 << 18), + // "probe" (bit 19): health-check traffic. Deliberately *not* InternalCallFlag, which also decides + // queuing - see IsCallerFacing. + ProbeFlag = (CommandFlags)(1 << 19); protected RedisCommand command; @@ -92,7 +95,15 @@ internal const CommandFlags | CommandFlags.NoScriptCache | MaskRetryCategory // caller may override the retry category... | CommandServerSpecific // ...and the server-specific flag - | NoFlushFlag; // we'll allow this one even though not advertised + | NoFlushFlag // we'll allow this one even though not advertised + // ...and the probe flag, which *has* to survive this + // whitelist: health-check probes reach the pipeline + // through the public API (HealthCheckContext. + // ProbeFlags), so it arrives as caller-supplied flags + // or not at all. A caller passing it deliberately only + // opts their own command out of endpoint-idleness + // accounting, which is harmless. + | ProbeFlag; private IResultBox? resultBox; @@ -271,6 +282,32 @@ internal void WithHighIntegrity(uint value) public bool IsFireAndForget => (Flags & CommandFlags.FireAndForget) != 0; public bool IsInternalCall => (Flags & InternalCallFlag) != 0; + /// + /// Whether this is health-check traffic, issued by a + /// rather than by a caller. + /// + public bool IsProbe => (Flags & ProbeFlag) != 0; + + /// + /// Whether somebody outside the library is waiting on this message. + /// + /// + /// The question idleness actually wants to ask. Our own traffic - handshakes, heartbeats, + /// autoconfigure, health probes - is work we chose to do, so counting it makes a server look busy + /// precisely because we are looking at it, and an endpoint that can never be retired is the result + /// (see the endpoint-retirement work). + /// + /// A probe is a separate bit from on purpose. The internal-call flag also + /// decides *queuing*: an internal call bypasses the backlog + /// (PhysicalBridge.TryPushToBacklog) and is queued while disconnected regardless of the backlog + /// policy (QueueOrFailMessage). Bypassing the backlog would make a health check unable to + /// observe the one thing it most needs to - a bridge whose queue is not draining - and that signal is + /// what drives failover in a geo-redundant deployment. So probes are excluded from *accounting* without + /// being given internal-call *routing*. + /// + /// + public bool IsCallerFacing => (Flags & (InternalCallFlag | ProbeFlag)) == 0; + public IResultBox? ResultBox => resultBox; public abstract int ArgCount { get; } // note: over-estimate if necessary diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 9baa29d20..a739e7df8 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -390,6 +390,14 @@ internal void KeepAlive(bool forceRun = false) msg.SetSource(ResultProcessor.Tracer, null); break; case ConnectionType.Subscription: + // Normally the PING - observed against a 7.0 server, answered with the two-element array + // pong rather than +PONG (OnResponseFrame's IsArrayPong is what keeps that out of the + // out-of-band path so it still matches this message). The UNSUBSCRIBE fallback is not just + // for pre-3.0 servers: the condition also fails when PING is disabled or renamed in the + // CommandMap, or fronted by something that does not support it. + // + // Neither needs SetInternalCall here: the common path below flags whatever the switch + // produced, so any tracer added later is covered without remembering to. if (commandMap.IsAvailable(RedisCommand.PING) && features.PingOnSubscriber) { msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.PING); @@ -673,8 +681,12 @@ internal void OnHeartbeat(bool ifConnectedOnly) // This is an "always" check - we always want to evaluate a dead connection from a non-responsive sever regardless of the need to heartbeat above var totalTimeoutThisHeartbeat = asyncTimeoutThisHeartbeat + syncTimeoutThisHeartbeat; - bool deadConnectionOnAsync = asyncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds * 4); - bool deadConnectionOnSync = syncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.TimeoutMilliseconds * 4); + // note these track the *effective* timeout: this heuristic is derived from the + // command timeout, so if relaxation says "be patient for 60s" while this says + // "dead after 4x5s", we would tear down the connection relaxation was protecting. + // Socket-level failure detection is untouched and still notices a dead server. + bool deadConnectionOnAsync = asyncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (ServerEndPoint.GetEffectiveTimeoutMilliseconds(tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds ?? 0) * 4); + bool deadConnectionOnSync = syncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (ServerEndPoint.GetEffectiveTimeoutMilliseconds(tmp.BridgeCouldBeNull?.Multiplexer.TimeoutMilliseconds ?? 0) * 4); if (deadConnectionOnAsync || deadConnectionOnSync) { // If we've received *NOTHING* on the pipe in 4 timeouts worth of time and we're timing out commands, issue a connection failure so that we reconnect @@ -1019,10 +1031,29 @@ private void StartBacklogProcessor() /// Crawls from the head of the backlog queue, consuming anything that should have timed out /// and pruning it accordingly (these messages will get timeout exceptions). /// + /// + /// Whether this bridge owes a *caller* anything, ignoring our own internal traffic. + /// + internal bool HasCallerWork() + { + if (physical?.HasCallerMessagesAwaitingResponse() == true) return true; + + foreach (var message in _backlog) // snapshot enumeration; a concurrent queue is fine to walk + { + if (message.IsCallerFacing) return true; + } + return false; + } + private void CheckBacklogForTimeouts() { var now = Environment.TickCount; - var timeout = _singleWriter.TimeoutMilliseconds; + + // deliberately *not* _singleWriter.TimeoutMilliseconds, which is the write-lock acquisition + // timeout: that is about contention between writers, and relaxing it during maintenance is not + // something anybody asked for. This is the age at which a queued command is considered timed out, + // relaxed while the server has announced a disruption. + var timeout = ServerEndPoint.GetEffectiveTimeoutMilliseconds(Multiplexer.TimeoutMilliseconds); // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. @@ -1303,7 +1334,7 @@ public bool HasPendingCallerFacingItems() { foreach (var item in _backlog) // non-consuming, thread-safe, etc { - if (!item.IsInternalCall) return true; + if (item.IsCallerFacing) return true; } } return physical?.HasPendingCallerFacingItems() ?? false; @@ -1763,6 +1794,33 @@ internal void SimulateConnectionFailure(SimulatedFailureType failureType) physical?.SimulateConnectionFailure(failureType); } + /// + /// Drops the current connection so that a fresh one is established. + /// + /// + /// Deliberately just a dispose: that routes through RecordConnectionFailed to + /// , which reconnects immediately by itself, so there is no new lifecycle + /// here to get wrong. Used by the MOVING handoff, where the point is to choose *when* the + /// connection is replaced - after the replacement address is known - rather than waiting for the server + /// to close it and re-resolving to whatever DNS says at that moment, which has been measured as still + /// naming the node being retired. + /// + internal bool RecycleConnection(string reason) + { + var current = physical; + if (current is null) return false; + + Multiplexer.Trace($"recycling {ConnectionType} connection: {reason}", ToString()); + + // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises + // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and + // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first + // also performs the disconnect, so the Dispose below is cleanup. + current.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + current.Dispose(); + return true; + } + internal RedisCommand? GetActiveMessage() => Volatile.Read(ref _activeMessage)?.Command; } } diff --git a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs new file mode 100644 index 000000000..03e74f875 --- /dev/null +++ b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs @@ -0,0 +1,304 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using RESPite; +using RESPite.Messages; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +internal sealed partial class PhysicalConnection +{ + /// + /// Reads a maintenance notification and raises it as an event; observation only. + /// + /// + /// Parsed leniently on purpose. These frames are dispatched here *before* anything reads element 1 as a + /// channel name, because element 1 is a sequence number - so a malformed one must be swallowed here, and + /// never fall through to the command matcher, where it would take a reply belonging to something else. + /// The shapes are (per the contract, all elements after the type being scalars): + /// + /// MOVING seq time endpoint + /// MIGRATING seq time shards, FAILING_OVER seq time shards + /// MIGRATED seq shards, FAILED_OVER seq shards + /// SMIGRATING seq slots, SMIGRATED seq slots + /// + /// Integers may arrive as : or as a bulk string, trailing elements are explicitly allowed, and the + /// endpoint may be an explicit null. So the type decides *whether* a time is expected rather than the + /// content deciding it - a slot list of "123" must not be mistaken for a duration - but a notification + /// that omits or adds one is still accepted. + /// + private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, PushKind kind, ref RespReader reader) + { + _readStatus = ReadStatus.MaintenanceNotification; + + // at most three elements follow the type in any defined shape; anything beyond that is ignored + string? e1 = null, e2 = null, e3 = null; + List? migrations = null; + int count = 0; + while (reader.SafeTryMoveNext()) + { + count++; + if (!reader.IsScalar) + { + // SMIGRATED nests: [type, seq, [[source, target, slots], ...]]. Everything else is flat, and + // nesting there means a frame we do not understand - so the guard stays for those, since + // guessing at an unknown shape is how a parser starts inventing data + if (kind is PushKind.SlotMigrated or PushKind.SlotMigrating && migrations is null) + { + // note the reader has to be moved *past* the aggregate: enumerating the children does not + // advance it, so without this the loop walks back into the triplets we just read and + // mistakes them for further top-level elements + migrations = ReadSlotMigrations(ref reader); + continue; + } + + Trace($"{kind}: non-scalar element {count}"); + return OutOfBandResult.Handled; + } + + var element = reader.IsNull ? null : reader.ReadString(); + switch (count) + { + case 1: e1 = element; break; + case 2: e2 = element; break; + case 3: e3 = element; break; + } + } + + var type = ToNotificationType(kind); + + // A missing or unreadable sequence id is *not* fatal. The specs say every shape carries one, but + // go-redis - which runs against real servers - length-checks these frames at two elements and reads + // no sequence number at all for the shard notifications. So a client that drops a frame for want of a + // seq is stricter than one that demonstrably works. Without it we lose only dedup, which is our own + // invention anyway; the disruption being announced is the part that matters. + long? sequenceId = TryParseInt64(e1, out var parsedSequenceId) ? parsedSequenceId : null; + if (sequenceId is null) + { + OnMaintenanceNotificationDropped(type, $"no readable sequence id in a {count + 1}-element frame; continuing without dedup"); + } + + long? timeSeconds = null; + string? payload; + if (CarriesTime(type)) + { + if (TryParseInt64(e2, out var seconds)) + { + timeSeconds = seconds; + payload = e3; + } + else + { + // tolerated: a server that omits the time it was supposed to send + payload = e3 ?? e2; + } + } + else if (count >= 3 && TryParseInt64(e2, out var seconds)) + { + // tolerated the other way: a time on a notification the contract says has none + timeSeconds = seconds; + payload = e3; + } + else + { + payload = e3 ?? e2; + } + + var server = BridgeCouldBeNull?.ServerEndPoint; + EndPoint? newEndPoint = null; + if (type == MaintenanceNotificationType.Moving && !string.IsNullOrEmpty(payload)) + { + // "?" and an empty host are the contract's placeholders for "no address", and are no more + // dialable than the null form; they must never be taken to mean "the server that told us" + newEndPoint = ParseMigrationEndPoint(payload); + if (newEndPoint is null) + { + Trace($"{kind}: no usable endpoint in '{payload}'"); + } + } + + var time = timeSeconds is { } value ? TimeSpan.FromSeconds(value) : (TimeSpan?)null; + var raw = Describe(kind, sequenceId, timeSeconds, payload); + Trace($"maintenance notification: {raw}"); + OnDetailLog($"maintenance notification: {raw}"); + + // relax before reporting: the event handler is consumer code, and the window should already be open + // by the time anyone sees the notification that opened it + if (server is not null) + { + if (IsWindowOpening(type)) + { + var isNew = server.OnMaintenanceWindowOpened(type, sequenceId, time); + + // ...and MOVING alone means "this endpoint is going away", which is worth acting on rather than + // waiting to be disconnected. + // + // Only when the notification is *new*, and that is load-bearing rather than tidy. A server + // re-sends MOVING to a connection that opts in while the window is still open - measured, and + // the handoff replaces the connection, so acting on the repeat is a feedback loop: recycle, + // reconnect, get told again, recycle. It produced twelve recycles from one event on a real + // deployment before this guard. The per-server sequence dedup already knew it was a repeat; the + // handoff simply was not asking. + if (isNew && type == MaintenanceNotificationType.Moving) + { + server.OnMovingAnnounced(time, newEndPoint, this); + } + } + else if (IsWindowClosing(type)) + { + server.OnMaintenanceWindowClosed(type, sequenceId); + + // ...and if slots moved away from us, learn the new topology rather than waiting to be told + // by a -MOVED. Scoped and jittered inside OnSlotsMigratedAway; see its remarks for why this + // is the cluster family only + if (type == MaintenanceNotificationType.SlotMigrated && migrations is not null) + { + server.OnSlotsMigratedAway(migrations); + } + } + } + + // Per-server work above, one event below: relaxation is per-connection and every connection is told, + // but a consumer wants one callback per logical event rather than one per proxy that mentioned it + if (muxer.TryClaimMaintenanceEvent(type, sequenceId)) + { + var evt = new PushMaintenanceEvent(type, sequenceId ?? 0, server?.EndPoint, time, newEndPoint, payload, raw, migrations); + muxer.OnServerMaintenanceEvent(evt); + } + else + { + Trace($"{kind} seq {sequenceId} already reported by another node; not raising again"); + } + + return OutOfBandResult.Handled; + } + + /// + /// Reads the nested [[source, target, slots], ...] form of a cluster slot-migration notification. + /// + /// + /// A malformed triplet is skipped rather than abandoning the whole notification - the same choice go-redis + /// makes, and the right one: the other triplets are still actionable, and one bad entry should not lose a + /// migration we could have applied. The slot list is a flat comma-and-range string inside each triplet. + /// + private List ReadSlotMigrations(ref RespReader reader) + { + var results = new List(); + var outer = reader.AggregateChildren(); + while (outer.MoveNext()) + { + var triplet = outer.Value; + if (!triplet.IsAggregate) + { + Trace("slot migration: expected a triplet"); + continue; + } + + string? source = null, target = null, slots = null; + int index = 0; + var inner = triplet.AggregateChildren(); + while (inner.MoveNext()) + { + var child = inner.Value; + var text = child.IsScalar && !child.IsNull ? child.ReadString() : null; + switch (index++) + { + case 0: source = text; break; + case 1: target = text; break; + case 2: slots = text; break; + } + } + + if (index < 3) + { + Trace($"slot migration: {index}-element triplet, skipped"); + continue; + } + + var ranges = SlotRange.TryParseList(slots, out var parsed) ? parsed : []; + results.Add(new ClusterSlotMigration(ParseMigrationEndPoint(source), ParseMigrationEndPoint(target), ranges, slots)); + } + + // leave the caller's reader after the aggregate, so it can carry on with any trailing elements + outer.MovePast(out reader); + return results; + } + + /// + /// Parses one end of a migration, treating the placeholder forms as "not named" rather than as an error. + /// + private static EndPoint? ParseMigrationEndPoint(string? value) + => string.IsNullOrEmpty(value) || value == "?" || !Format.TryParseEndPoint(value, out var parsed) || IsPlaceholderEndPoint(parsed) + ? null + : parsed; + + private static bool IsPlaceholderEndPoint(EndPoint endpoint) => endpoint switch + { + DnsEndPoint dns => dns.Port == 0 || dns.Host is "" or "?", + IPEndPoint ip => ip.Port == 0, + _ => false, + }; + + private void OnMaintenanceNotificationDropped(MaintenanceNotificationType type, string reason) + { + // never fatal: a notification we cannot read is a diagnostic, not a protocol failure - the frame has + // been consumed either way, so the connection is not at risk + Trace($"dropped {type} notification: {reason}"); + OnDetailLog($"dropped {type} notification: {reason}"); + } + + private static bool TryParseInt64(string? value, out long result) + => long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out result); + + /// + /// Whether this notification announces a disruption starting (or still running). + /// + /// + /// is an opener with no closer: its window can only end + /// by the deadline the server gave us, or - later - by the handoff completing. + /// + private static bool IsWindowOpening(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Moving + or MaintenanceNotificationType.Migrating + or MaintenanceNotificationType.FailingOver + or MaintenanceNotificationType.SlotMigrating; + + /// + /// Whether this notification announces that a disruption has finished. + /// + private static bool IsWindowClosing(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.FailedOver + or MaintenanceNotificationType.SlotMigrated; + + /// + /// Whether the contract gives this notification a time element. + /// + private static bool CarriesTime(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Moving + or MaintenanceNotificationType.Migrating + or MaintenanceNotificationType.FailingOver; + + private static MaintenanceNotificationType ToNotificationType(PushKind kind) => kind switch + { + PushKind.Moving => MaintenanceNotificationType.Moving, + PushKind.Migrating => MaintenanceNotificationType.Migrating, + PushKind.Migrated => MaintenanceNotificationType.Migrated, + PushKind.FailingOver => MaintenanceNotificationType.FailingOver, + PushKind.FailedOver => MaintenanceNotificationType.FailedOver, + PushKind.SlotMigrating => MaintenanceNotificationType.SlotMigrating, + PushKind.SlotMigrated => MaintenanceNotificationType.SlotMigrated, + _ => MaintenanceNotificationType.None, + }; + + private static string Describe(PushKind kind, long? sequenceId, long? timeSeconds, string? payload) + { + var sb = new System.Text.StringBuilder(kind.ToString().ToUpperInvariant()); + sb.Append(" seq=").Append(sequenceId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "?"); + if (timeSeconds is { } seconds) sb.Append(" time=").Append(seconds).Append('s'); + if (payload is not null) sb.Append(' ').Append(payload); + return sb.ToString(); + } +} diff --git a/src/StackExchange.Redis/PhysicalConnection.Read.cs b/src/StackExchange.Redis/PhysicalConnection.Read.cs index e65941021..91efe09c9 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Read.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Read.cs @@ -508,11 +508,36 @@ internal enum PushKind PUnsubscribe, [AsciiHash("sunsubscribe")] SUnsubscribe, + + // the maintenance-notification family; these are *not* pub/sub - element 1 is a sequence number + // rather than a channel, so they must be dispatched before anything reads a channel name. The specs + // write them uppercase while the pub/sub kinds above are lowercase, hence the case-insensitive match + [AsciiHash("MOVING")] + Moving, + [AsciiHash("MIGRATING")] + Migrating, + [AsciiHash("MIGRATED")] + Migrated, + [AsciiHash("FAILING_OVER")] + FailingOver, + [AsciiHash("FAILED_OVER")] + FailedOver, + [AsciiHash("SMIGRATING")] + SlotMigrating, + [AsciiHash("SMIGRATED")] + SlotMigrated, } internal static partial class PushKindMetadata { - [AsciiHash] + /// + /// Identifies a push frame from its first element. + /// + /// + /// Case-insensitive: the pub/sub kinds are lowercase on the wire and the maintenance kinds are + /// uppercase, and no specification anywhere is careful about it - so don't bake in an assumption. + /// + [AsciiHash(CaseSensitive = false)] internal static partial bool TryParse(ReadOnlySpan value, out PushKind result); } @@ -563,6 +588,12 @@ private OutOfBandResult OnOutOfBand(ReadOnlySpan payload, ref IMemoryOwner if (!reader.TryParseScalar(&PushKindMetadata.TryParse, out kind)) kind = PushKind.None; } + if (kind is >= PushKind.Moving and <= PushKind.SlotMigrated) + { + // not pub/sub: dispatch before anything tries to read element 1 as a channel name + return OnMaintenanceNotification(muxer, kind, ref reader); + } + RedisChannel.RedisChannelOptions channelOptions = kind switch { PushKind.PMessage or PushKind.PSubscribe or PushKind.PUnsubscribe => RedisChannel.RedisChannelOptions.Pattern, diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 1c4222584..c16cfa40a 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -811,6 +811,31 @@ internal void GetStormLog(StringBuilder sb) } } + /// + /// Whether any message awaiting a response was issued by a *caller* rather than by us. + /// + /// + /// The distinction matters wherever we ask "is anyone using this server": our own handshake, + /// autoconfigure and keep-alive traffic is not use, and treating it as such makes a server we are + /// probing look busy *because* we are probing it. + /// + /// A predicate rather than a count, because every caller only asks whether the answer is zero - and + /// this way the common case (a caller's message at the head of the queue) costs one iteration instead + /// of walking a queue that a stalled server can leave thousands of entries long. + /// + /// + internal bool HasCallerMessagesAwaitingResponse() + { + lock (_writtenAwaitingResponse) + { + foreach (var message in _writtenAwaitingResponse) + { + if (message.IsCallerFacing) return true; + } + } + return false; + } + /// /// Runs on every heartbeat for a bridge, timing out any commands that are overdue and returning an integer of how many we timed out. /// @@ -829,7 +854,9 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou { var server = bridge.ServerEndPoint; var multiplexer = bridge.Multiplexer; - var timeout = multiplexer.AsyncTimeoutMilliseconds; + + // relaxed while this server has announced a disruption; a floor, never a reduction + var timeout = server.GetEffectiveTimeoutMilliseconds(multiplexer.AsyncTimeoutMilliseconds); foreach (var msg in _writtenAwaitingResponse) { // We only handle async timeouts here, synchronous timeouts are handled upstream. @@ -1272,6 +1299,7 @@ internal enum ReadStatus ResetArena, ProcessBufferComplete, PubSubUnsubscribe, + MaintenanceNotification, NA = -1, } @@ -1287,7 +1315,7 @@ internal bool HasPendingCallerFacingItems() { foreach (var item in _writtenAwaitingResponse) { - if (!item.IsInternalCall) return true; + if (item.IsCallerFacing) return true; } } return false; diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 419c74dc2..4b87978f5 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -111,3 +111,68 @@ static StackExchange.Redis.BitFieldOperation.IncrementBy(StackExchange.Redis.Bit static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.Set(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset, long value, StackExchange.Redis.BitFieldOverflow overflow = StackExchange.Redis.BitFieldOverflow.Wrap) -> StackExchange.Redis.BitFieldOperation +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.set -> void +[SER010]StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Auto = 2 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Disabled = 0 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Enabled = 1 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Maintenance.PushMaintenanceEvent.ToString() -> string? +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailedOver = 5 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailingOver = 4 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrated = 3 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrating = 2 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Moving = 1 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.None = 0 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrated = 7 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrating = 6 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.EndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NewEndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NotificationType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Payload.get -> string? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SequenceId.get -> long +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Time.get -> System.TimeSpan? +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.AbortOnConnectFail.get -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.IsMatch(System.Net.EndPoint! endpoint) -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +[SER010]override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +StackExchange.Redis.Configuration.RedisCloudOptionsProvider +StackExchange.Redis.Configuration.RedisCloudOptionsProvider.RedisCloudOptionsProvider() -> void +virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.Name.get -> string? +override StackExchange.Redis.Configuration.AzureOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +[SER010]override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.RedisEnterpriseOptionsProvider() -> void +override StackExchange.Redis.Configuration.DefaultOptionsProvider.ToString() -> string! +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.set -> void +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan? +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedWindowMax.get -> System.TimeSpan? +[SER010]StackExchange.Redis.Availability.FaultContext.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.RedisConnectionException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.RedisTimeoutException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]override StackExchange.Redis.Maintenance.ClusterSlotMigration.ToString() -> string! +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.ClusterSlotMigration() -> void +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.RawSlots.get -> string? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Slots.get -> System.Collections.Generic.IReadOnlyList! +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Source.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Target.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SlotMigrations.get -> System.Collections.Generic.IReadOnlyList! +[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeFlags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.ConnectionFailureType.MaintenanceHandoff = 12 -> StackExchange.Redis.ConnectionFailureType diff --git a/src/StackExchange.Redis/RedisLiterals.cs b/src/StackExchange.Redis/RedisLiterals.cs index 12b85ea2f..d66cb841f 100644 --- a/src/StackExchange.Redis/RedisLiterals.cs +++ b/src/StackExchange.Redis/RedisLiterals.cs @@ -72,6 +72,7 @@ public static readonly RedisValue LIMIT = RedisValue.FromRaw("LIMIT"u8), LIST = RedisValue.FromRaw("LIST"u8), LT = RedisValue.FromRaw("LT"u8), + MAINT_NOTIFICATIONS = RedisValue.FromRaw("MAINT_NOTIFICATIONS"u8), MATCH = RedisValue.FromRaw("MATCH"u8), MALLOC_STATS = RedisValue.FromRaw("MALLOC-STATS"u8), MAX = RedisValue.FromRaw("MAX"u8), @@ -97,6 +98,8 @@ public static readonly RedisValue PATTERN = RedisValue.FromRaw("PATTERN"u8), PAUSE = RedisValue.FromRaw("PAUSE"u8), PERSIST = RedisValue.FromRaw("PERSIST"u8), + OFF = RedisValue.FromRaw("OFF"u8), + ON = RedisValue.FromRaw("ON"u8), PING = RedisValue.FromRaw("PING"u8), PREPARE = RedisValue.FromRaw("PREPARE"u8), PURGE = RedisValue.FromRaw("PURGE"u8), diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 2175294e8..f8eb06a0e 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -25,6 +25,7 @@ public static readonly ResultProcessor TrackSubscriptions = new TrackSubscriptionsProcessor(null), Tracer = new TracerProcessor(false), EstablishConnection = new TracerProcessor(true), + MaintenanceNotifications = new MaintenanceNotificationsProcessor(), BackgroundSaveStarted = new ExpectBasicStringProcessor(Literals.background_saving_started.Hash, startsWith: true), BackgroundSaveAOFStarted = new ExpectBasicStringProcessor(Literals.background_aof_rewriting_started.Hash, startsWith: true); @@ -3191,6 +3192,44 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } + /// + /// Handles the reply to the maintenance-notification opt-in, which we send speculatively: a server that + /// doesn't know the subcommand replies with an error, and that is an expected outcome rather than a + /// fault. So the error is absorbed here rather than going through the common error path, which would + /// raise an to the consumer for something we asked for + /// on their behalf. + /// + private sealed class MaintenanceNotificationsProcessor : ResultProcessor + { + public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) + { + reader.MovePastBof(); + var server = connection.BridgeCouldBeNull?.ServerEndPoint; + if (reader.IsError) + { + server?.OnMaintenanceNotificationsRefused(connection, reader.ReadString() ?? "declined"); + SetResult(message, false); + return true; + } + + if (reader.IsScalar && Literals.OK.Hash.IsCS(reader.TryGetSpan(out var span) ? span : reader.Buffer(stackalloc byte[16]))) + { + server?.OnMaintenanceNotificationsAccepted(); + SetResult(message, true); + return true; + } + + // anything else: treat as "not available" rather than a protocol fault; being liberal in what + // we accept matters more here than pinning an unverifiable reply shape + server?.OnMaintenanceNotificationsRefused(connection, $"unexpected reply: {reader.GetOverview()}"); + SetResult(message, false); + return true; + } + + protected override bool SetResultCore(PhysicalConnection connection, Message message, ref RespReader reader) + => throw new NotSupportedException(); // SetResult is fully overridden + } + private sealed class TracerProcessor(bool establishConnection) : ResultProcessor { public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs new file mode 100644 index 000000000..28f00e1b0 --- /dev/null +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -0,0 +1,591 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +internal sealed partial class ServerEndPoint +{ + private volatile bool _maintenanceNotificationsActive, _maintenanceNotificationsRequested; + private volatile string? _maintenanceNotificationsRefusal; + + /// + /// Whether this server has accepted our request for maintenance notifications on the current connection. + /// + /// + /// Per-connection state, so this is cleared at the start of every handshake and re-established from the + /// reply; the scope is deliberately the server rather than the multiplexer, so one lagging node doesn't + /// disable the feature for the whole deployment. + /// + internal bool MaintenanceNotificationsActive => _maintenanceNotificationsActive; + + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + private MaintenanceNotificationMode MaintenanceMode + => Multiplexer.RawConfig.MaintenanceNotifications; + + /// + /// Whether to ask this server for maintenance notifications during handshake. + /// + /// + /// The feature is RESP3-only: the notifications are out-of-band push frames on the connection that + /// carries commands, and a RESP2 connection can have the request accepted and then silently receive + /// nothing - so we don't ask unless we asked for RESP3. (Note that not asking is not the same as being + /// satisfied: under the reconcile below fails the + /// connection for exactly this case.) We can't know what was *negotiated* at write time + /// (the handshake is pipelined, and HELLO hasn't been answered yet), so this tests what we asked + /// for and settles it once the reply has been processed. + /// + private bool ShouldRequestMaintenanceNotifications(bool isInteractive, bool negotiateResp3) + => isInteractive + && negotiateResp3 + && MaintenanceMode != MaintenanceNotificationMode.Disabled + && Multiplexer.CommandMap.IsAvailable(RedisCommand.CLIENT); + + internal void OnMaintenanceNotificationsAccepted() => _maintenanceNotificationsActive = true; + + /// + /// The server declined our request. Recorded rather than acted on: whether that matters is a question for + /// , which sees the negotiated protocol too. + /// + internal void OnMaintenanceNotificationsRefused(PhysicalConnection connection, string reason) + { + _maintenanceNotificationsActive = false; + _maintenanceNotificationsRefusal = reason; + connection.OnDetailLog($"maintenance notifications refused: {reason}"); + } + + /// + /// Settles the feature for this connection now that the handshake is complete and the protocol is known. + /// + /// + /// The opt-in reply precedes the tracer on the same pipelined connection, so by the time this runs every + /// fact is in: whether we asked, what the server said, and what protocol we ended up on. + /// + private void ReconcileMaintenanceNotifications(PhysicalConnection connection) + { + bool resp3 = connection.Protocol is >= RedisProtocol.Resp3; + if (!resp3) + { + // whatever the server said about the opt-in, nothing can arrive on a RESP2 connection + _maintenanceNotificationsActive = false; + } + + if (_maintenanceNotificationsActive || MaintenanceMode != MaintenanceNotificationMode.Enabled) + { + return; + } + + // Enabled means required: no notifications, no connection. That includes a configuration that never + // got as far as asking - requiring a RESP3-only feature over RESP2 is a contradiction, and failing it + // is more useful than honouring half of it. Note the cross-client spec only calls for failing when + // the *server* errors; extending that to the RESP2 cases is ours, and deliberate + var reason = !resp3 + ? (_maintenanceNotificationsRequested ? "the connection negotiated RESP2" : "RESP3 was not requested") + : _maintenanceNotificationsRefusal ?? "the server did not accept the request"; + + connection.RecordConnectionFailed( + ConnectionFailureType.ProtocolFailure, + new RedisConnectionException(ConnectionFailureType.ProtocolFailure, CommandFlags.None, $"Maintenance notifications are enabled, but unavailable: {reason}", innerException: null)); + } + + // The relaxed-timeout window, expressed as a single deadline in Environment.TickCount terms so that it + // can be read without a lock from the heartbeat sweeps. Zero means "no window"; a deadline that computes + // to zero is nudged by a tick rather than complicating the sentinel. Comparisons are unchecked + // subtraction, as elsewhere in this type, so the ~49-day wrap is a non-event. + private int _relaxedDeadlineTicks; + + // the notification that last touched the window, reported on faults that happen inside it + private int _relaxedType; + + // Dedup state, per notification type. No specification defines the sequence ids, but they were observed on + // Enterprise 8.6.2 to be monotonic per database and shared across types, and to carry the same value on + // every node broadcasting a given event - so they identify the event, which is exactly what dedup needs. + // Keyed per type anyway: within a type the ids are still monotonic, and a per-type key cannot mistake one + // node's earlier event for a replay of another's later one. Allocated on first notification, so a server + // that never sees one pays nothing. + private long[]? _lastSequenceIds; + private bool[]? _haveSequenceIds; // zero is a real sequence number, so "unset" needs its own bit + private readonly object _maintenanceSync = new(); + + /// + /// Whether timeouts are currently relaxed for this server. + /// + internal bool IsMaintenanceRelaxed => GetRelaxedRemaining() > 0; + + /// + /// The notification in force for this server, or if no + /// window is open; reported on faults so that a timeout during a migration says so. + /// + internal MaintenanceNotificationType ActiveMaintenanceType + => GetRelaxedRemaining() > 0 ? (MaintenanceNotificationType)Volatile.Read(ref _relaxedType) : MaintenanceNotificationType.None; + + /// + /// The effective timeout for a command against this server: the configured value, or the relaxed value if + /// that is larger and a window is open. Relaxation is a floor and can never shorten a timeout. + /// + /// + /// Read from the timeout sweeps rather than stamped onto each message: both sweeps rely on head-of-line + /// ordering and stop at the first message that has not timed out, which per-message timeouts would + /// invalidate. The consequence is that relaxation applies to whatever is outstanding when a window opens, + /// and stops applying when it closes - which is one of the reasons the post-event tail exists. + /// + internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) + { + if (GetRelaxedRemaining() <= 0) return configuredMilliseconds; + + var relaxed = (int)Multiplexer.RawConfig.MaintenanceRelaxedTimeout.TotalMilliseconds; + return relaxed > configuredMilliseconds ? relaxed : configuredMilliseconds; + } + + /// + /// Milliseconds of relaxation left, or zero if none; clears an expired window as a side-effect. + /// + private int GetRelaxedRemaining() + { + var deadline = Volatile.Read(ref _relaxedDeadlineTicks); + if (deadline == 0) return 0; + + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining > 0) return remaining; + + // expired; clear it, but only if nobody has moved it on in the meantime + Interlocked.CompareExchange(ref _relaxedDeadlineTicks, 0, deadline); + return 0; + } + + /// + /// An announced disruption has started (or is still running): open or extend the relaxed window. + /// + /// + /// Whether this notification was new. A replay extends nothing and, importantly, must not be *acted* on - + /// see the handoff, where re-acting on a repeat is a feedback loop rather than merely wasted work. + /// + internal bool OnMaintenanceWindowOpened(MaintenanceNotificationType type, long? sequenceId, TimeSpan? time) + { + if (!TryClaimSequenceId(type, sequenceId)) return false; + + var config = Multiplexer.RawConfig; + var floor = config.MaintenanceRelaxedTimeout; + var cap = config.MaintenanceRelaxedWindowMax; + + // no time, or a time at or below zero, means "act now" rather than "ignore this": a connection that + // arrives mid-window is told what is left of it, and that can legitimately be negative + var duration = time is { } value && value > TimeSpan.Zero ? value : floor; + if (duration < floor) duration = floor; + if (duration > cap) duration = cap; + + Volatile.Write(ref _relaxedType, (int)type); + ExtendRelaxedWindow(duration, $"{type} for {duration.TotalSeconds}s"); + return true; + } + + /// + /// An announced disruption has finished: replace the remaining window with the post-event tail. + /// + /// + /// Replace rather than extend-or-shorten. The server has told us the operation completed, so whatever it + /// previously said about duration is stale - but the tail still applies, because completion is when every + /// other client that received the same notification re-engages. + /// + internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? sequenceId) + { + if (!TryClaimSequenceId(type, sequenceId)) return; + + Volatile.Write(ref _relaxedType, (int)type); + var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; + if (tail <= TimeSpan.Zero) + { + Volatile.Write(ref _relaxedDeadlineTicks, 0); + Multiplexer.Trace($"{type}: relaxation ended", ToString()); + return; + } + + var deadline = NudgeFromZero(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); + Volatile.Write(ref _relaxedDeadlineTicks, deadline); + Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); + } + + private int _handoffInFlight, _handoffRecycles; + private volatile string? _lastHandoffOutcome; + + /// + /// What the last MOVING handoff decided, and why. + /// + /// + /// Recorded rather than only traced because Multiplexer.Trace is [Conditional("VERBOSE")] - it + /// compiles away in any normal build, so a handoff that misbehaves in production would leave no trace at + /// all. This is the minimum that survives: what was decided, readable afterwards. + /// + internal string? LastHandoffOutcome => _lastHandoffOutcome; + + /// How many times a handoff has replaced this server's connections. + internal int HandoffRecycles => Volatile.Read(ref _handoffRecycles); + + /// + /// Acts on a MOVING: find where to go, then get off this connection before we are pushed. + /// + /// + /// The value here is entirely in the *timing*. Without it we already survive a MOVING - the socket + /// closes and we reconnect - but the reconnect happens when the server decides, and re-resolves to whatever + /// DNS says at that moment, which has been measured as still naming the node being retired. Measured on a + /// real deployment: MOVING arrives about six seconds in, DNS moves somewhere between four and + /// nineteen seconds later, and the socket closes seventeen to nineteen seconds after the notification. So + /// the window exists to be *used*, and using it means waiting for DNS to move and then choosing the moment. + /// + /// Fire-and-forget by design: this runs while the connection it concerns is still serving commands, and the + /// notification is delivered on the read loop, which must not wait for a DNS poll. + /// + /// + internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalConnection connection) + { + // One at a time per server. A rolling operation delivers one MOVING per connection, so a second one + // arriving while a handoff is in flight is a repeat or a much later event; either way, starting a + // second poll against the same endpoint achieves nothing. + if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) + { + Multiplexer.Trace("MOVING: a handoff is already in flight", ToString()); + return; + } + + var budget = window is { } value && value > TimeSpan.Zero + ? value + : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor + var current = (connection.VolatileSocket?.RemoteEndPoint as IPEndPoint)?.Address; + + _ = HandoffAsync(budget, successor, current); + } + + private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? currentAddress) + { + try + { + // Spread the fleet, but only by a fraction of the window - see MaintenanceHandoff.GetJitter for why + // a flat delay is wrong here. + var jitter = MaintenanceHandoff.GetJitter(window, RandomFor(this)); + if (jitter > TimeSpan.Zero) await Task.Delay(jitter).ForAwait(); + + var remaining = window - jitter; + var decision = await MaintenanceHandoff.DecideAsync( + EndPoint, + successor, + currentAddress, + remaining, + pollInterval: TimeSpan.FromSeconds(1), // records carry a 5s TTL, so this is several looks per record + resolve: Multiplexer.AddressResolver, + log: message => Multiplexer.Trace(message, ToString())).ForAwait(); + + Multiplexer.Trace($"MOVING: {decision}", ToString()); + _lastHandoffOutcome = decision.ToString(); + switch (decision.Action) + { + case HandoffAction.Recycle: + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + break; + case HandoffAction.Reconfigure: + // A named successor is a different endpoint, so this is a topology question. Note no + // observed deployment has ever named one, so this path has never run outside a test. + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, "moving names a successor"); + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + break; + default: + // Nothing to do is a legitimate outcome, not a failure: the server closes the socket, the + // reconnect re-resolves, and the relaxed window covers the gap. + break; + } + } + catch (Exception ex) + { + Multiplexer.OnInternalError(ex, EndPoint); + } + finally + { + Volatile.Write(ref _handoffInFlight, 0); + } + } + + /// + /// Lets in-flight caller work finish, then replaces the connections. + /// + /// + /// Draining first because the socket is still working: anything already written may still be answered, and + /// dropping it would fail commands that were about to succeed. Bounded by what is left of the window, + /// because the server closes the socket at the end of it regardless - so anything not drained by then was + /// going to fail either way, and draining strictly dominates. + /// + /// Both bridges, not just the one that was told. The measured blast radius is the *node*: four connections + /// to one proxy, differing only in handshake, all closed simultaneously, and only the ones that had opted in + /// were warned. + /// + /// + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) + { + var watch = ValueStopwatch.StartNew(); + while (HasCallerWork() && watch.ElapsedMilliseconds < budget.TotalMilliseconds) + { + await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); + } + + var drained = !HasCallerWork(); + var recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); + if (recycled) Interlocked.Increment(ref _handoffRecycles); + Multiplexer.Trace( + $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" + + (drained ? " (drained)" : " (still busy; the window ran out)"), + ToString()); + } + + [ThreadStatic] + private static Random? _random; + + /// + /// A per-thread , seeded so that two processes handing off at once do not pick the same + /// jitter. + /// + private static Random RandomFor(ServerEndPoint server) => + _random ??= new Random(Environment.TickCount ^ server.GetHashCode()); + + private void ExtendRelaxedWindow(TimeSpan duration, string cause) + { + var candidate = NudgeFromZero(unchecked(Environment.TickCount + (int)duration.TotalMilliseconds)); + while (true) + { + var current = Volatile.Read(ref _relaxedDeadlineTicks); + + // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer + // MIGRATING window must not cut it short + if (current != 0 && unchecked(candidate - current) <= 0) return; + if (Interlocked.CompareExchange(ref _relaxedDeadlineTicks, candidate, current) == current) + { + Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); + return; + } + } + } + + /// + /// Zero is the "no window" sentinel, so a deadline that lands on it moves by a tick. + /// + private static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; + + /// + /// Whether this notification is new, per the conservative dedup described on . + /// + private bool TryClaimSequenceId(MaintenanceNotificationType type, long? sequenceId) + { + // no id, no dedup - but the notification still counts. Dropping an announced disruption because we + // could not read a field whose meaning nobody has defined would be the wrong way round + if (sequenceId is not { } id) return true; + + var index = (int)type; + lock (_maintenanceSync) + { + var ids = _lastSequenceIds ??= new long[MaintenanceNotificationTypeCount]; + var have = _haveSequenceIds ??= new bool[MaintenanceNotificationTypeCount]; + if ((uint)index >= (uint)ids.Length) return true; // unknown type: don't dedup what we can't index + + // The "have we seen one" bit is separate because zero is a real sequence number - observed on + // Enterprise 8.6.2 as the first event of a chain (`>4 $6 MOVING :0 :15 _`). Treating a stored zero + // as "unset", which an earlier cut did, quietly disabled dedup for whichever notification happened + // to open the chain. + // + // note "<=", not "<": a replay carries the id we already acted on + if (have[index] && id <= ids[index]) + { + Multiplexer.Trace($"{type}: ignoring replayed sequence id {id}", ToString()); + return false; + } + + ids[index] = id; + have[index] = true; + return true; + } + } + + private const int MaintenanceNotificationTypeCount = 8; // None + the seven notification types + + /// + /// How long to smear a notification-triggered topology refresh over. + /// + /// + /// Not configurable, deliberately. The relaxed-window durations are options because their right value is + /// deployment-specific and we invented them; this is a fixed small smear that exists only so that a fleet + /// which was all told the same thing at the same instant does not all query the same node at the same + /// instant. Nobody needs to tune it, and every option is public surface to keep. Promote it if that turns + /// out to be wrong. + /// + /// Note already declines while one refresh is in + /// flight, so the *local* storm is handled; this is purely about the fleet. + /// + /// + private static readonly int MaintenanceRefreshJitterMilliseconds = 1000; + + // 1 while a jittered refresh is scheduled and has not yet started + private int _refreshPending; + + /// + /// Whether this server is one of the sources in a slot-migration delta - i.e. whether the movement being + /// described is movement away from us. + /// + /// + /// Every node in the cluster reports the same movements, so the sender is not implicitly a source and most + /// notifications describe somebody else. Acting only on our own is what stops a fleet re-reading topology + /// because one shard moved somewhere unrelated - go-redis makes the same choice. + /// + /// Resolution goes through the identity map rather than comparing endpoints directly: a node answers to + /// its address and to its announced hostname, and the delta may name either. + /// + /// + private bool IsSourceOf(IReadOnlyList migrations) + { + foreach (var migration in migrations) + { + if (migration.Source is { } source && ReferenceEquals(Multiplexer.TryResolveServerEndPoint(source), this)) + { + return true; + } + } + + return false; + } + + /// + /// Refreshes the topology after slots have moved away from this server, once the fleet has had a moment to + /// spread out. + /// + /// + /// Deliberately only for the cluster family: MIGRATED and FAILED_OVER arrive in proxied + /// deployments where the client addresses a single endpoint, so there is no topology for a refresh to + /// learn - they get relaxation and nothing else. Endpoints left serving nothing are handled by the + /// existing absence-based pruning, which a refresh feeds; there is deliberately no second, faster + /// retirement path here. + /// + internal void OnSlotsMigratedAway(IReadOnlyList migrations) + { + if (migrations.Count == 0 || !IsSourceOf(migrations)) return; + + // Coalesce *before* the jitter, not after. ReconfigureIfNeeded declines only while a refresh is + // actually in flight, and the jitter spreads a burst of notifications out far enough that each one + // completes before the next begins - so relying on that alone turns ten notifications into ten + // topology passes. Found by the test that counts them. + if (Interlocked.CompareExchange(ref _refreshPending, 1, 0) != 0) + { + Multiplexer.Trace("topology refresh already pending; folding this notification into it", ToString()); + return; + } + + var cause = $"slots migrated from {Format.ToString(EndPoint)}"; + Multiplexer.Trace($"{cause}; refreshing topology after jitter", ToString()); + + // fire-and-forget: we are on the read loop, and the refresh must not block it + _ = RefreshAfterJitterAsync(cause, migrations); + } + + private async Task RefreshAfterJitterAsync(string cause, IReadOnlyList migrations) + { + try + { + await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(MaintenanceRefreshJitterMilliseconds)).ForAwait(); + + // deliberately *after* the delay - see the remarks on ResubscribeStrandedShardedChannels + ResubscribeStrandedShardedChannels(migrations); + + // released before the refresh runs, not after: anything that arrives from here on describes a + // state this pass may not have seen, and deserves its own pass + Volatile.Write(ref _refreshPending, 0); + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, cause); + } + catch (Exception ex) + { + // a refresh we failed to start is a missed optimization, not a fault: the next -MOVED still works + Volatile.Write(ref _refreshPending, 0); + Multiplexer.Trace($"topology refresh after {cause} failed: {ex.Message}", ToString()); + } + } + + /// + /// Re-establishes sharded subscriptions that the slot movement stranded and nothing else recovered. + /// + /// + /// Mostly belt-and-braces: a server that migrates a slot also sends an unsolicited SUNSUBSCRIBE, + /// and already resubscribes on that. This adds two things. + /// It is *pre-emptive* where SMIGRATED arrives first, and it *covers* the case where the + /// unsolicited unsubscribe never arrives or is lost - in which case the only other signal is a message + /// that silently stops being delivered, which nothing detects. + /// + /// It also knows which slots moved, so only the affected channels are touched rather than everything + /// subscribed on this server. + /// + /// + /// Why this runs after the jitter, and only for a subscription connected nowhere. An earlier cut ran + /// it immediately, on the reasoning that a silently-unsubscribed subscriber is a correctness problem while a + /// stale slot map is only a round trip. Measured against a fake that emits the realistic sequence, that + /// produced an extra resubscribe per channel - because the unsolicited unsubscribe had already started one, + /// and mid-flight the subscription still looked like ours. Being a genuine *fallback* means acting only + /// once the other path has had its chance, which costs a stranded subscription up to the jitter in + /// recovery time and costs nothing when it was not needed. + /// + /// + /// Measured against the fake's realistic sequence: four (re)subscribes with notifications off, five with + /// them on. The extra one is this fallback acting on a subscription the unsolicited-unsubscribe path left + /// attached to nothing - i.e. it is the feature working, not duplicate work. One extra attempt per + /// stranded channel, not one per notification. + /// + /// + /// Note it resubscribes via this server, not the migration target, reusing + /// unchanged: the outgoing node is the one we know has + /// the new route, and sending there follows the redirect. The target is named in the notification and + /// could be dialled directly, but it may be a node we have never seen, or named in a form we cannot dial, + /// and the redirect path is the one already proven by the SUNSUBSCRIBE case. + /// + /// + private void ResubscribeStrandedShardedChannels(IReadOnlyList migrations) + { + var subscriptions = Multiplexer.GetSubscriptions(); + if (subscriptions.IsEmpty) return; + + var strategy = Multiplexer.ServerSelectionStrategy; + foreach (var pair in subscriptions) + { + var channel = pair.Key; + + // ordinary pub/sub is not slot-bound, so a slot moving says nothing about it + if (!channel.IsSharded) continue; + + var slot = strategy.HashSlot(channel); + if (slot == ServerSelectionStrategy.NoSlot || !IsInMigratedRange(migrations, slot)) continue; + + // Skip only if it is attached somewhere *else* - that means the other path already moved it. Still + // attached to us is the pre-emptive case (the notification beat the unsubscribe, and the + // subscription is now stale); attached nowhere is the stranded case. Both need acting on, and + // distinguishing them from "already handled" is the whole job of this check. + var subscription = pair.Value; + var current = subscription.GetAnyCurrentServer(); + if (current is not null && !ReferenceEquals(current, this)) + { + Multiplexer.Trace($"slot {slot} moved, and {channel} has already moved with it; leaving it alone", ToString()); + continue; + } + + Multiplexer.Trace($"slot {slot} moved; resubscribing {channel}", ToString()); + Multiplexer.DefaultSubscriber.ResubscribeToServer(subscription, channel, this, cause: "smigrated"); + } + } + + private static bool IsInMigratedRange(IReadOnlyList migrations, int slot) + { + foreach (var migration in migrations) + { + foreach (var range in migration.Slots) + { + if (slot >= range.From && slot <= range.To) return true; + } + } + + return false; + } +} diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 17d37a5c0..49c408e06 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -331,7 +331,22 @@ internal bool IsIdle() => !Multiplexer.ServerSelectionStrategy.OwnsAnySlot(this) && (subscription?.SubscriptionCount ?? 0) == 0 && (interactive?.SubscriptionCount ?? 0) == 0 - && GetOutstandingCount() == 0; + && !HasCallerWork(); + + /// + /// Whether a *caller* is waiting on anything here, which is the only kind of work that should stop us + /// retiring a server. + /// + /// + /// Deliberately not , which counts everything. A node the topology has + /// stopped listing still receives our autoconfigure probes on every pass, and nothing answers them, so + /// they accumulate in its backlog: measured at ~170 per pass, growing without bound. Counting those + /// made the node look busy *because* we were looking for it, so it could never be retired - the + /// precondition defeated itself in exactly the case pruning exists for. Keep-alive traffic has the same + /// property, and is excluded by the same test, since both set the internal-call flag. + /// + internal bool HasCallerWork() + => interactive?.HasCallerWork() == true || subscription?.HasCallerWork() == true; /// /// Work this server still owes an answer on: written-and-awaiting-response, plus anything queued in @@ -360,18 +375,23 @@ internal async Task RetireAsync(string reason, TimeSpan drainTimeout, ILogger? l SetUnselectable(UnselectableFlags.Retiring); log?.LogInformationRetiringServer(new(EndPoint), reason); + // Drain what a *caller* is waiting for, not everything outstanding. Our own probes to a node that + // has gone away will never be answered, so draining on the total means always waiting out the full + // timeout before letting go - measured: a departed node accumulates our autoconfigure traffic + // indefinitely (500+ and climbing), so the drain never once completed early. Callers are who the + // drain exists for; nobody is waiting on our keep-alives. // Stopwatch rather than TickCount64: the latter does not exist on the down-level targets var watch = ValueStopwatch.StartNew(); - int outstanding; - while ((outstanding = GetOutstandingCount()) > 0 && watch.ElapsedMilliseconds < drainTimeout.TotalMilliseconds) + while (HasCallerWork() && watch.ElapsedMilliseconds < drainTimeout.TotalMilliseconds) { await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } - if (outstanding > 0) + if (HasCallerWork()) { - // deliberately reported: an abandoned command is exactly what a caller will be asking about - log?.LogInformationRetiringServerAbandoned(new(EndPoint), outstanding); + // deliberately reported: an abandoned command is exactly what a caller will be asking about. + // The count is the total, since that is what is actually being dropped on the floor + log?.LogInformationRetiringServerAbandoned(new(EndPoint), GetOutstandingCount()); } Dispose(); @@ -926,6 +946,11 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; + + if (connection is not null && bridge == interactive) + { + ReconcileMaintenanceNotifications(connection); + } if (bridge == subscription || isResp3) { // Note: this MUST be fire and forget, because we might be in the middle of a Sync processing @@ -1241,6 +1266,10 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) // forget what the previous connection's HELLO told us; re-established below, if this one repeats it // (the subscription handshake is deliberately left out of this: it doesn't do the discovery step) RoleKnownFromHello = false; + + // likewise per-connection: re-armed from this handshake's reply, if we ask + _maintenanceNotificationsActive = _maintenanceNotificationsRequested = false; + _maintenanceNotificationsRefusal = null; } // HELLO serves two purposes: negotiating RESP3, and reporting details we would otherwise need INFO or @@ -1332,6 +1361,19 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) msg = Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.ID); msg.SetInternalCall(); await WriteDirectOrQueueFireAndForgetAsync(connection, msg, autoConfig ??= ResultProcessor.AutoConfigureProcessor.Create(log)).ForAwait(); + + if (ShouldRequestMaintenanceNotifications(isInteractive, negotiateResp3)) + { + _maintenanceNotificationsRequested = true; + // speculative in the same way as the AUTH above: we don't yet know what HELLO negotiated, + // so we ask whenever we asked for RESP3, and ReconcileMaintenanceNotifications sorts out a + // downgrade once the reply has been processed. A bare ON is explicitly valid: the server + // then picks the endpoint type, which is what we want until we derive one ourselves. + log?.LogInformationRequestingMaintenanceNotifications(new(this), MaintenanceMode); + msg = Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON); + msg.SetInternalCall(); + await WriteDirectOrQueueFireAndForgetAsync(connection, msg, ResultProcessor.MaintenanceNotifications).ForAwait(); + } } var bridge = connection.BridgeCouldBeNull; diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 587af0622..e799f510a 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -42,6 +42,7 @@ + @@ -59,6 +60,7 @@ + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs new file mode 100644 index 000000000..b904632e2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The OSS cluster notification family against a real cluster: SMIGRATING, SMIGRATED, and what +/// the client is supposed to do about them. +/// +/// +/// Every /slot-migrate trigger requires oss_cluster: true with all-master-shards, so this +/// scenario family is the cluster half of the feature - the half whose reactions (re-reading the slot map, +/// re-subscribing stranded sharded channels) have until now only been exercised against the in-process fake. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "cluster-family")] +public class ClusterFamilyScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + /// + /// Slot migrations, in the four shapes the injector can produce. + /// + /// + /// The effects differ in what happens to the *endpoint list*, which is the thing a client has to keep up + /// with: remove-add retires a node and introduces one, remove only retires, add only + /// introduces, and slot-shuffle moves slots between nodes that both stay. So they cover the four + /// ways a slot map can go stale, and only one of them (shuffle) leaves the endpoint set alone. + /// + /// + /// Only remove here. The other effects need a database the setup leg will not provision, and get + /// their own fixture below rather than skipping: add and slot-shuffle need a node holding + /// several shards, and remove-add is not in /slot-migrate/setup's effect enum at all. + /// + [Theory] + [InlineData("remove", "migrate")] + public async Task SlotMigrationIsAnnouncedAndActedOn(string effect, string trigger) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "slot-migrate", effect, trigger, log.WriteLine, + setupTrigger: "reshard", cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + ScenarioSupport.RequireEffectIsAchievable(scenario, effect); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + foreach (var migration in push.SlotMigrations) + { + log.WriteLine($" slots {migration.RawSlots}: {migration.Source} -> {migration.Target}"); + } + } + }; + + var endpointsBefore = conn.GetEndPoints().Length; + log.WriteLine($"connected as {conn.GetServer(conn.GetEndPoints()[0]).ServerType} across {endpointsBefore} endpoint(s)"); + + // keys spread across slots, so a migration of any shard moves something we are actually using + var db = conn.GetDatabase(); + var keys = Enumerable.Range(0, 32).Select(i => (RedisKey)$"fi-{effect}-{i}").ToArray(); + foreach (var key in keys) await db.StringSetAsync(key, "before"); + + clock.Restart(); + await ScenarioSupport.FireOrSkipAsync(scenario, effect, cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports finished"); + + // Keep using the connection while things move: a slot map that has gone stale shows up as MOVED + // redirections, and the point of the feature is that we learn the new one rather than eating them. + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + int reads = 0, failures = 0; + while (clock.Elapsed < deadline) + { + foreach (var key in keys) + { + try + { + await db.StringGetAsync(key); + reads++; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + failures++; + if (failures <= 3) log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s read failed: {ex.Message}"); + } + } + + await Task.Delay(500, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s); {reads} reads, {failures} failures; " + + $"endpoints {endpointsBefore} -> {conn.GetEndPoints().Length}"); + + // A slot migration is announced to every proxy, so any connection should see it. What is asserted is + // only that we were told and understood it - the *reaction* is asserted separately below, because a + // migration that moves no slot we hold would legitimately change nothing here. + Assert.NotEmpty(events); + Assert.All(events, e => Assert.NotEqual(MaintenanceNotificationType.None, e.NotificationType)); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.SlotMigrating or MaintenanceNotificationType.SlotMigrated); + + // Any SMIGRATED that carried a payload must have parsed into something usable; an empty list would + // mean we saw the frame and threw the contents away. + foreach (var migrated in events.Where(e => e.NotificationType == MaintenanceNotificationType.SlotMigrated)) + { + if (!string.IsNullOrEmpty(migrated.Payload) || migrated.SlotMigrations.Count > 0) + { + Assert.NotEmpty(migrated.SlotMigrations); + Assert.All(migrated.SlotMigrations, m => Assert.NotNull(m.Source)); + } + } + } + + // and the client is still serving keys across the new topology + Assert.True(reads > 0, "no reads succeeded at all during the migration"); + foreach (var key in keys) Assert.Equal("before", await db.StringGetAsync(key)); + } + + [Fact] + public async Task ShardedSubscriptionSurvivesASlotMigration() + { + // D5's resubscription, against a real cluster. A sharded channel is bound to the node owning its slot, + // so a migration strands the subscription: the server sends an unsolicited SUNSUBSCRIBE and stops + // delivering. Recovering that is the client's job, and until now only the fake has tested it. + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "slot-migrate", "remove", "migrate", log.WriteLine, + setupTrigger: "reshard", cancellationToken: cancellationToken); + Assert.NotNull(scenario.Database); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(scenario.Database.GetClientConfig(fixture.Environment)); + var subscriber = conn.GetSubscriber(); + + // several channels, because the migration moves the slots of one node: with one channel the test would + // usually be asserting that nothing happened to it + var channels = Enumerable.Range(0, 16).Select(i => RedisChannel.Sharded($"fi-shard-{i}")).ToArray(); + var received = new int[channels.Length]; + for (int i = 0; i < channels.Length; i++) + { + var index = i; + await subscriber.SubscribeAsync(channels[i], (_, _) => Interlocked.Increment(ref received[index])); + } + + Assert.True(await DeliversEverywhereAsync(subscriber, channels, received, cancellationToken), + "every sharded channel should deliver before the migration, or the test proves nothing"); + log.WriteLine($"all {channels.Length} sharded channels delivering before the migration"); + + await ScenarioSupport.FireOrSkipAsync(scenario, "remove", cancellationToken); + + // The recovery is deliberately allowed to be slow: it is driven by a jittered topology refresh and, as a + // fallback, by a delayed resubscription sweep. What matters is that it happens without the caller doing + // anything, not that it is instant. + var recovered = await DeliversEverywhereAsync(subscriber, channels, received, cancellationToken, timeoutSeconds: 90); + log.WriteLine($"delivery after migration: {(recovered ? "all channels" : "INCOMPLETE")}"); + Assert.True(recovered, "sharded subscriptions should deliver again once the topology settles"); + } + + /// + /// Publishes to every channel and waits until each one has delivered at least one more message. + /// + /// + /// Publishing repeatedly rather than once: the publish itself routes by slot, so during a migration an + /// individual publish can land on a node that no longer owns the slot. Retrying is what a real caller would + /// do, and it keeps the test measuring *delivery* rather than the timing of one message. + /// + private async Task DeliversEverywhereAsync( + ISubscriber subscriber, + RedisChannel[] channels, + int[] received, + CancellationToken cancellationToken, + int timeoutSeconds = 20) + { + var baseline = channels.Select((_, i) => Volatile.Read(ref received[i])).ToArray(); + var deadline = DateTime.UtcNow.AddSeconds(timeoutSeconds); + + while (DateTime.UtcNow < deadline) + { + for (int i = 0; i < channels.Length; i++) + { + if (Volatile.Read(ref received[i]) > baseline[i]) continue; + try + { + await subscriber.PublishAsync(channels[i], "ping"); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // expected mid-migration; the retry is the point + } + } + + await Task.Delay(500, cancellationToken); + if (channels.Select((_, i) => Volatile.Read(ref received[i]) > baseline[i]).All(x => x)) return true; + } + + var missing = channels.Where((_, i) => Volatile.Read(ref received[i]) <= baseline[i]).Select(c => c.ToString()); + log.WriteLine($" not delivering: {string.Join(", ", missing)}"); + return false; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs new file mode 100644 index 000000000..2146492a5 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// An OSS cluster with packed shards, so add and slot-shuffle have something to move. +/// +public sealed class DenseClusterFixture() : FaultInjectorFixture(DatabaseShape.OssClusterDense); + +/// +/// The slot migrations the scenario setup legs cannot provision for. +/// +/// +/// These looked like an environment limitation - the injector refuses them with "No node with multiple shards +/// found" - and they are not: the setup leg provisions one shard per node, so there is never a node to take a +/// shard *from*. Six shards with dense placement over three nodes gives two per node, and both effects +/// then run. The lesson generalises: a scenario that setup cannot arrange is still reachable by provisioning +/// the database ourselves and driving the run leg by bdb_id. +/// +/// No scenario teardown here, deliberately. Teardown deletes the database, and this database belongs to the +/// fixture, which deletes it itself; and migrate excludes no nodes, so there is nothing to restore. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "cluster-family")] +public class DenseClusterScenarioTests(DenseClusterFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("add")] + [InlineData("slot-shuffle")] + [InlineData("remove-add")] + public async Task SlotMigrationIsAnnouncedAndActedOn(string effect) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + log.WriteLine($"{effect} against {database}"); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + foreach (var migration in push.SlotMigrations) + { + log.WriteLine($" slots {migration.RawSlots}: {migration.Source} -> {migration.Target}"); + } + } + }; + + var db = conn.GetDatabase(); + var keys = Enumerable.Range(0, 32).Select(i => (RedisKey)$"fi-dense-{effect}-{i}").ToArray(); + foreach (var key in keys) await db.StringSetAsync(key, "before"); + + clock.Restart(); + var query = new Dictionary + { + ["effect"] = effect, + ["trigger"] = "migrate", + ["bdb_id"] = database.BdbId.ToString(), + }; + await fixture.Injector.PostScenarioAsync("slot-migrate", leg: null, query, cancellationToken: cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports finished"); + + int reads = 0, failures = 0; + var deadline = clock.Elapsed + TimeSpan.FromSeconds(20); + while (clock.Elapsed < deadline) + { + foreach (var key in keys) + { + try + { + await db.StringGetAsync(key); + reads++; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + failures++; + } + } + + await Task.Delay(500, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s); {reads} reads, {failures} failures"); + Assert.NotEmpty(events); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.SlotMigrating + or MaintenanceNotificationType.SlotMigrated); + } + + Assert.True(reads > 0, "no reads succeeded during the migration"); + foreach (var key in keys) Assert.Equal("before", await db.StringGetAsync(key)); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs new file mode 100644 index 000000000..de33a1d44 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Security.Cryptography.X509Certificates; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Checks that the environment's TLS material belongs to the cluster we are about to talk to. +/// +/// +/// Environments get re-provisioned, and the certificates do not always follow. Observed 2026-09-01: the folder +/// held a server certificate for *.marcgravell-test-46be1d08... while the live cluster was +/// marcgravell-test-e21cd75d... - a leftover from a previous provision, three days older than the +/// env_output.json beside it. +/// +/// Without this check, that presents as a TLS handshake failure, which reads like a client bug and is not one: +/// the certificate genuinely does not cover the name being dialled, and refusing it is correct +/// (TrustIssuer tolerates chain errors only, so a name mismatch fails - as it should). Detecting it here +/// turns half an hour of certificate archaeology into a skip that names both clusters, and it costs nothing +/// because it happens before a database is provisioned. +/// +/// +internal static class CertificateSanity +{ + /// + /// Skips when the environment's certificates were issued for a different cluster. + /// + public static void RequireCertificatesMatchThisCluster(FaultInjectorEnvironment environment, Action log) + { + var clusterName = environment.Cluster?.ClusterName; + if (clusterName is null) return; // nothing to compare against; let the connect speak for itself + + // the server certificate the environment generated, if it left one behind + var leafPath = Path.Combine(environment.ConfigDirectory.FullName, "redis.crt"); + if (!File.Exists(leafPath)) return; + + var leaf = X509CertificateLoader.LoadCertificateFromFile(leafPath); + var names = ReadDnsNames(leaf); + log($"environment server certificate covers [{string.Join(", ", names)}]; cluster is '{clusterName}'"); + + if (!names.Any(name => CoversClusterEndpoints(name, clusterName))) + { + Assert.Skip( + $"the environment's TLS certificates cover [{string.Join(", ", names)}] but this cluster is " + + $"'{clusterName}' - they are left over from an earlier provision, so a TLS test here would " + + "only be measuring the mismatch. Re-provision with certificate generation enabled to run it."); + } + } + + /// + /// Every DNS name a certificate carries, not just the first. + /// + /// + /// GetNameInfo returns one name, which is not enough: these certificates carry both a wildcard and + /// the bare cluster name, and which one comes back decides the answer. Reading the SAN extension properly is + /// the difference between a check that works and one that skips a perfectly good environment - as the first + /// version of this did. + /// + private static List ReadDnsNames(X509Certificate2 certificate) + { + foreach (var extension in certificate.Extensions) + { + if (extension is X509SubjectAlternativeNameExtension san) + { + return [.. san.EnumerateDnsNames()]; + } + } + + var subject = certificate.GetNameInfo(X509NameType.DnsName, forIssuer: false); + return subject is null ? [] : [subject]; + } + + /// + /// Whether a certificate name covers the endpoints of a given cluster. + /// + /// + /// The hosts actually dialled are *database* endpoints - redis-13500.<cluster> - so a wildcard + /// whose parent is the cluster name is exactly right, even though (correctly) it does not match the bare + /// cluster name itself. An exact SAN for the cluster name also counts, since that is what the REST API is + /// reached by. + /// + private static bool CoversClusterEndpoints(string certificateName, string clusterName) + { + if (certificateName.StartsWith("*.", StringComparison.Ordinal)) + { + return string.Equals(certificateName[2..], clusterName, StringComparison.OrdinalIgnoreCase); + } + + return string.Equals(certificateName, clusterName, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs new file mode 100644 index 000000000..c9565bb79 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The cluster's own REST API on port 9443, for the few facts the fault injector does not expose. +/// +/// +/// Reads only. Anything that *changes* state goes through the injector so it is recorded as a job with an +/// action id - which is also how the console works, and it means a scenario can be reconstructed afterwards +/// from the injector's history rather than from somebody's memory. +/// +public sealed class ClusterRestClient : IDisposable +{ + private readonly HttpClient _http; + + public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, string? certificateAuthorityPath) + { + var handler = new HttpClientHandler(); + + if (certificateAuthorityPath is not null) + { + // The cluster's management certificate is self-signed per environment, like the proxy ones. Pin to + // the CA in the config directory rather than accepting anything: this channel carries credentials. + var issuer = X509CertificateLoader.LoadCertificateFromFile(certificateAuthorityPath); + handler.ServerCertificateCustomValidationCallback = (_, certificate, chain, _) => + { + if (certificate is null || chain is null) return false; + chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.Add(issuer); + chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + return chain.Build(certificate); + }; + } + + _http = new HttpClient(handler) { BaseAddress = credentials.RestUrl, Timeout = TimeSpan.FromSeconds(30) }; + var basic = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credentials.Username}:{credentials.Password}")); + _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basic); + } + + public void Dispose() => _http.Dispose(); + + /// + /// Every database on the cluster, as (bdb_id, name). + /// + public async Task> ListDatabasesAsync() + { + using var response = await _http.GetAsync("/v1/bdbs?fields=uid,name"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var results = new List<(int, string)>(); + foreach (var bdb in document.RootElement.EnumerateArray()) + { + if (bdb.TryGetProperty("uid", out var uid) && bdb.TryGetProperty("name", out var name) + && uid.TryGetInt32(out var id) && name.GetString() is { } text) + { + results.Add((id, text)); + } + } + + return results; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs new file mode 100644 index 000000000..b9778e213 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The database shapes this suite provisions, described declaratively. +/// +/// +/// Shape is the axis that matters, and it is not cosmetic: measurements on 2026-08-28 showed the number of A +/// records a hostname carries follows actual proxy *placement*, giving 2 for all-nodes, 3 for +/// all-master-shards and 1 for single - and the handoff takes a different branch depending on +/// whether a live sibling address exists. So the shapes below are the reason to provision databases from inside +/// the tests at all: they turn that into a matrix axis instead of a manual sweep. +/// +public sealed record DatabaseShape( + string Label, + bool OssCluster = false, + string? ProxyPolicy = null, + bool Tls = false, + bool Replication = false, + int ShardCount = 1, + string? EndpointType = null, + string ShardsPlacement = "sparse", + string IpType = "external") +{ + /// A proxied standalone database: the shape MOVING actually fires on. + public static readonly DatabaseShape ProxiedStandalone = new("proxied-standalone", ProxyPolicy: "single"); + + /// Multiple proxies, so the hostname carries several A records and a sibling always exists. + public static readonly DatabaseShape MultiProxy = new("multi-proxy", ProxyPolicy: "all-master-shards", ShardCount: 2); + + /// OSS cluster API: the family that emits SMIGRATING/SMIGRATED instead. + public static readonly DatabaseShape OssClusterApi = new("oss-cluster", OssCluster: true, ProxyPolicy: "all-master-shards", ShardCount: 2); + + /// + /// An OSS cluster whose shards are packed, so some node holds more than one. + /// + /// + /// Exists to reach the add and slot-shuffle migrations, which need a node holding several + /// shards to have something to move. The scenario setup legs cannot produce that - they provision one shard + /// per node - so those effects looked like an environment limitation until this was tried: six shards with + /// dense placement over three nodes gives two per node, and both effects then run. + /// + /// Note it also demonstrates the placement-versus-policy point from the other direction: packed masters mean + /// all-master-shards advertises a *single* address, because the count follows placement. + /// + /// + public static readonly DatabaseShape OssClusterDense = new( + "oss-cluster-dense", OssCluster: true, ProxyPolicy: "all-master-shards", ShardCount: 6, ShardsPlacement: "dense"); + + /// + /// TLS with hostname-advertised endpoints - the documented coverage gap. + /// + /// + /// endpoint_type: ip is the interesting counterpart: with addresses advertised, a verifying client + /// cannot check identity on the targets it is told to use, because the proxy certificate carries DNS names + /// and no IP SAN. That pairing is the thing no in-process harness can honestly reproduce. + /// + public static readonly DatabaseShape TlsWithHostnames = new("tls-hostnames", OssCluster: true, Tls: true, ShardCount: 2, EndpointType: "hostname"); + + /// + /// The create_database parameters for this shape. + /// + /// + /// Goes inside a database_config wrapper when it reaches the injector - a flat payload is rejected + /// with "Invalid parameter 'database_config': got None". + /// + /// Built as data rather than as a typed record, because these are the injector's wire names and its schema + /// declares parameters as untyped. The names here are no longer guesses: they match the environment's + /// own bdb_config.json and the dbconfig the injector itself publishes as a trigger requirement + /// (GET /topology-change-standalone?effect=...), which is the authoritative list. + /// + /// + public Dictionary ToCreateParameters(string name, int port) + { + var parameters = new Dictionary + { + ["name"] = name, + ["port"] = port, + ["memory_size"] = 134_217_728, // 128MB, matching what the injector asks for in its own requirements + ["eviction_policy"] = "volatile-lru", + ["replication"] = Replication, + ["sharding"] = ShardCount > 1, + ["shards_count"] = ShardCount, + ["shards_placement"] = ShardsPlacement, + ["oss_cluster"] = OssCluster, + }; + + if (ShardCount > 1) + { + // Required whenever sharding is on: Redis Enterprise rejects the database outright with + // "Invalid sharding configuration: missing shard_key_regex". These two patterns are the standard + // pair - an explicit hash tag if present, otherwise the whole key - and are what the environment's + // own bdb_config.json uses. + parameters["shard_key_regex"] = new[] + { + new Dictionary { ["regex"] = ".*\\{(?.*)\\}.*" }, + new Dictionary { ["regex"] = "(?.*)" }, + }; + } + + // only when asked for: a database with no tls_mode serves plaintext, and setting it here without + // certificates in place produces a database nothing can connect to + if (Tls) parameters["tls_mode"] = "enabled"; + + if (OssCluster) + { + // Which addresses CLUSTER SLOTS advertises, and it is not cosmetic: the default is *internal*, so a + // cluster database created without this advertises VPC-private addresses. A client outside the VPC + // then connects to the seed, discovers nodes at 10.x, and cannot reach any of them - which is + // exactly how the first dense-cluster run failed. Distinct from EndpointType (ip versus hostname), + // which is about identity rather than routing. + parameters["oss_cluster_api_preferred_ip_type"] = IpType; + } + + if (OssCluster && EndpointType is not null) + { + // Two distinct axes, easily conflated. "endpoint_type" is ip-versus-hostname - what CLUSTER SLOTS + // advertises, and therefore whether a verifying TLS client can check identity on the targets it is + // given. "ip_type" is internal-versus-external, which is about routing rather than identity. + parameters["oss_cluster_api_preferred_endpoint_type"] = EndpointType; + } + + if (ProxyPolicy is not null) parameters["proxy_policy"] = ProxyPolicy; + + return parameters; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs new file mode 100644 index 000000000..3040ea891 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A database the environment already provisioned, read from endpoints.json. +/// +/// +/// The counterpart to creating our own. Provisioning gives control over the shape, which is what the matrix +/// needs; this gives a run against whatever the environment template made, which is what you want when the +/// question is "does any of this work at all" rather than "does it work for shape X". +/// +/// Note endpoints.json carries more than expected: each entry's raw_endpoints includes +/// proxy_policy, oss_cluster_api_preferred_endpoint_type and the address list behind the DNS +/// name. So the facts that decide client behaviour are mostly here, and the cluster REST API is only needed for +/// the explicit oss_cluster flag and the client-certificate settings. +/// +/// +public sealed record ExistingDatabase( + string Key, + int BdbId, + string Host, + int Port, + bool Tls, + string? Username, + string? Password, + string? ProxyPolicy, + string? EndpointType, + IReadOnlyList Addresses) +{ + /// + /// How many addresses the hostname is expected to resolve to. + /// + /// + /// The measured driver of handoff behaviour: with more than one address a live sibling always exists, so a + /// handoff steps sideways immediately; with one, it has to wait for the record to move. Tests that care + /// should assert on this rather than on the policy name, because the count follows actual proxy + /// *placement* - an all-master-shards database whose shards share a node advertises one address. + /// + public int AdvertisedAddressCount => Addresses.Count; + + public override string ToString() => $"{Key} ({Host}:{Port}, bdb {BdbId}, {ProxyPolicy ?? "?"}, {AdvertisedAddressCount} addr)"; + + /// + /// Reads every database in the environment's endpoints.json, keyed as that file keys them. + /// + public static Dictionary ReadAll(FaultInjectorEnvironment environment) + { + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + var path = Path.Combine(environment.ConfigDirectory.FullName, "endpoints.json"); + if (!File.Exists(path)) return results; + + using var document = JsonDocument.Parse(File.ReadAllText(path)); + foreach (var entry in document.RootElement.EnumerateObject()) + { + if (entry.Value.ValueKind != JsonValueKind.Object) continue; + if (TryRead(entry.Name, entry.Value, out var database)) results[entry.Name] = database; + } + + return results; + } + + private static bool TryRead(string key, JsonElement element, out ExistingDatabase database) + { + database = null!; + if (!element.TryGetProperty("bdb_id", out var bdbId) || !bdbId.TryGetInt32(out var id)) return false; + + // "endpoints" holds "host:port" or a redis:// URI depending on how the environment was templated + string? host = null; + int port = 0; + if (element.TryGetProperty("endpoints", out var endpoints) && endpoints.ValueKind == JsonValueKind.Array + && endpoints.GetArrayLength() > 0 && endpoints[0].GetString() is { } first) + { + var text = first; + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) text = text[(scheme + 3)..]; + var colon = text.LastIndexOf(':'); + if (colon > 0 && int.TryParse(text[(colon + 1)..], out port)) host = text[..colon]; + } + + string? proxyPolicy = null, endpointType = null; + var addresses = new List(); + if (element.TryGetProperty("raw_endpoints", out var raw) && raw.ValueKind == JsonValueKind.Array + && raw.GetArrayLength() > 0) + { + var head = raw[0]; + proxyPolicy = ReadString(head, "proxy_policy"); + endpointType = ReadString(head, "oss_cluster_api_preferred_endpoint_type"); + host ??= ReadString(head, "dns_name"); + if (port == 0 && head.TryGetProperty("port", out var rawPort)) rawPort.TryGetInt32(out port); + + if (head.TryGetProperty("addr", out var addr) && addr.ValueKind == JsonValueKind.Array) + { + foreach (var item in addr.EnumerateArray()) + { + if (item.GetString() is { } address) addresses.Add(address); + } + } + } + + if (host is null || port == 0) return false; + + database = new ExistingDatabase( + key, + id, + host, + port, + element.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, + ReadString(element, "username"), + ReadString(element, "password"), + proxyPolicy, + endpointType, + addresses); + return true; + } + + private static string? ReadString(JsonElement element, string name) + => element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null; + + /// + /// Connection options for this database, with maintenance notifications required. + /// + public ConfigurationOptions GetClientConfig(FaultInjectorEnvironment environment, MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + User = string.Equals(Username, "default", StringComparison.OrdinalIgnoreCase) ? null : Username, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = mode, + AbortOnConnectFail = false, + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Tls) + { + options.Ssl = true; + options.SslHost = Host; + var caPath = environment.CertificateAuthorityPath + ?? throw new InvalidOperationException($"{Key} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}"); + options.TrustIssuer(caPath); + } + + return options; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs new file mode 100644 index 000000000..c6ebe4453 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Runs against the databases the environment template already created, rather than provisioning new ones. +/// +/// +/// The cheap tier, and the right one to start from: it needs none of the create_database schema, so it +/// answers "does the client work against a real deployment" before anything depends on parameter names that are +/// documented only as prose. +/// +public class ExistingDatabaseFixture : IAsyncLifetime +{ + private FaultInjectorClient? _injector; + private string? _skipReason; + + public IReadOnlyDictionary Databases { get; private set; } + = new Dictionary(); + + public FaultInjectorEnvironment Environment => + FaultInjectorEnvironment.Current ?? throw new InvalidOperationException("no environment; call RequireAvailable first"); + + public FaultInjectorClient Injector => + _injector ?? throw new InvalidOperationException("no injector; call RequireAvailable first"); + + /// Skips the calling test when there is nothing to talk to. + public void RequireAvailable() + { + if (_skipReason is not null) Assert.Skip(_skipReason); + } + + /// + /// The named database, or a skip if this environment does not have one. + /// + /// + /// A skip rather than a failure, deliberately: which databases exist is a property of the environment + /// template somebody chose, so a template without an oss_cluster database is a reason not to run the + /// cluster tests, not evidence of a bug. + /// + public ExistingDatabase Require(string key) + { + RequireAvailable(); + if (!Databases.TryGetValue(key, out var database)) + { + Assert.Skip($"this environment has no '{key}' database (found: {string.Join(", ", Databases.Keys)})"); + } + + return database!; + } + + /// + /// Where fixture-level diagnostics go. Deliberately collected rather than written to + /// : a fixture has no test output helper, and anything it prints to the console is + /// easily lost - which is how a silently failing setup call survived a day of use. + /// + public List SetupLog { get; } = []; + + private void log(string message) + { + SetupLog.Add(message); + Console.WriteLine(message); + } + + public async ValueTask InitializeAsync() + { + if (FaultInjectorEnvironment.Current is null) + { + _skipReason = FaultInjectorEnvironment.UnavailableReason ?? "no fault-injector environment configured"; + return; + } + + if (!FaultInjectorEnvironment.IsEnabled) + { + _skipReason = "set E2E_SCENARIO_TESTS=true to run against a real deployment"; + return; + } + + _injector = new FaultInjectorClient(FaultInjectorEnvironment.Current.InjectorUrl); + Databases = ExistingDatabase.ReadAll(FaultInjectorEnvironment.Current); + await EnsureMaintenanceNotificationsEnabledAsync(); + } + + public ValueTask DisposeAsync() + { + _injector?.Dispose(); + return default; + } + + /// + /// Turns on the cluster-level maintenance-notification flags. + /// + /// + /// A different thing from the per-connection opt-in, and both are required: the cluster flag decides whether + /// CLIENT MAINT_NOTIFICATIONS exists at all, and the command opts one connection in. Without this a + /// run fails at connect - correctly, since the tests ask for + /// - but for a reason that has nothing to do with the + /// client, so it is worth setting rather than diagnosing. + /// + /// Through the injector rather than a direct REST PUT, so the change leaves an action id behind and + /// shows up in the injector's history like everything else. Best-effort: if the action type is not + /// available on this build, the connect failure that follows says so clearly enough. + /// + /// + private async Task EnsureMaintenanceNotificationsEnabledAsync() + { + // nested under "config", not flattened - the same shape create_database wants for "database_config". + // A flat payload is rejected with "Invalid parameter 'config': got None", and because this call is + // best-effort it failed *silently* for a whole day of testing without anybody noticing. The impact was + // small: the environment templates enable these flags at provision time, so this is a safety net rather + // than the mechanism, and the tests were passing on their own merits. It matters for an environment + // provisioned without them, where the alternative is every test failing at connect and blaming the + // client for a server-side setting. + var parameters = new Dictionary + { + ["config"] = new Dictionary + { + ["client_maint_notifications"] = true, // proxy-routed databases + ["oss_cluster_client_maint_notifications"] = true, // oss_cluster databases + }, + }; + + try + { + await Injector.RunActionAsync("update_cluster_config", parameters, timeout: TimeSpan.FromMinutes(2)); + log("cluster maintenance-notification flags enabled"); + } + catch (Exception ex) + { + // Still best-effort - a cluster may not support the flags at all - but no longer silent: it goes to + // the test output, where the connect failure that follows can be attributed to it. + log($"could not set cluster maintenance-notification flags: {ScenarioSupport.Summarize(ex.Message)}"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs new file mode 100644 index 000000000..6b1b18b5a --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Everything this suite needs to reach a real deployment, discovered from one directory. +/// +/// +/// One path is the whole configuration, deliberately. That directory is the one mounted into the fault +/// injector as /app/config and the one docker compose up is run from, so it already holds the +/// compose file, the cluster credentials (env_output.json), the CA certificate, and - once databases +/// exist - endpoints.json. Asking for anything else would mean the person who provisioned the +/// environment has to hand-carry facts into the test run, which is the error-prone part. +/// +/// Point at it with SER_FI_CONFIG_DIR; FI_CONSOLE_CONFIG_DIR is honoured as a fallback so this +/// runs against the same directory as the fault-injector console with no extra setup. +/// +/// +public sealed class FaultInjectorEnvironment +{ + private const string ConfigDirVariable = "SER_FI_CONFIG_DIR"; + private const string ConsoleConfigDirVariable = "FI_CONSOLE_CONFIG_DIR"; + private const string InjectorUrlVariable = "FAULT_INJECTION_API_URL"; + private const string EnabledVariable = "E2E_SCENARIO_TESTS"; + private const string DefaultInjectorUrl = "http://127.0.0.1:20324"; + + /// + /// The environment for this run, or null when none is configured. + /// + public static FaultInjectorEnvironment? Current { get; } = Discover(); + + /// + /// Why there is no environment, for a skip message that says something useful. + /// + public static string? UnavailableReason { get; private set; } + + private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) + { + ConfigDirectory = configDirectory; + InjectorUrl = injectorUrl; + } + + public DirectoryInfo ConfigDirectory { get; } + + public Uri InjectorUrl { get; } + + /// + /// The CA certificate that signs the proxy certificates, for . + /// + /// + /// The certificates are self-signed per environment, so trusting the issuer is how a TLS test validates + /// anything at all. Note what this is *not*: a switch that disables validation. A test that cannot find the + /// CA fails rather than falling back to trusting everything, because a TLS test that silently stops + /// checking identity is worse than no TLS test - it reports success for the one thing it exists to catch. + /// + public string? CertificateAuthorityPath { get; private set; } + + /// + /// Cluster credentials for the REST API on port 9443, when env_output.json carries them. + /// + /// + /// Needed for the facts endpoints.json does not carry - notably oss_cluster and the + /// advertised endpoint type - which matter because they decide which notification family a database can + /// even emit. Tests that provision their own database know what they asked for and need none of this. + /// + public ClusterCredentials? Cluster { get; private set; } + + private static FaultInjectorEnvironment? Discover() + { + var path = Environment.GetEnvironmentVariable(ConfigDirVariable) + ?? Environment.GetEnvironmentVariable(ConsoleConfigDirVariable); + + if (string.IsNullOrWhiteSpace(path)) + { + UnavailableReason = $"no fault-injector environment: set {ConfigDirVariable} to the directory holding env_output.json"; + return null; + } + + var directory = new DirectoryInfo(path); + if (!directory.Exists) + { + // configured but wrong is a mistake worth reporting, not a reason to quietly do nothing + UnavailableReason = $"{ConfigDirVariable} points at '{path}', which does not exist"; + return null; + } + + var url = Environment.GetEnvironmentVariable(InjectorUrlVariable); + var injectorUrl = Uri.TryCreate(url, UriKind.Absolute, out var parsed) ? parsed : new Uri(DefaultInjectorUrl); + + var result = new FaultInjectorEnvironment(directory, injectorUrl) + { + CertificateAuthorityPath = FindCertificateAuthority(directory), + }; + result.Cluster = ReadClusterCredentials(directory); + return result; + } + + /// + /// Whether the caller actually meant to run destructive tests against a real deployment. + /// + /// + /// Two gates rather than one, and they mean different things. Without a config directory there is nothing + /// to talk to, which is the ordinary state for everybody else and skips. With a directory but without + /// E2E_SCENARIO_TESTS=true we also skip, because these tests create and delete databases and nobody + /// should trip that by running the full traversal. What must *not* happen is a third state where the + /// environment is configured, meant, and broken, yet the run still reports success - see + /// . + /// + public static bool IsEnabled => + string.Equals(Environment.GetEnvironmentVariable(EnabledVariable), "true", StringComparison.OrdinalIgnoreCase); + + private static string? FindCertificateAuthority(DirectoryInfo directory) + { + // names seen across the environment templates, most specific first + foreach (var candidate in new[] { "ca.pem", "ca.crt", "proxy_cert.pem", "redislabs_ca.pem" }) + { + var path = Path.Combine(directory.FullName, candidate); + if (File.Exists(path)) return path; + } + + return null; + } + + private static ClusterCredentials? ReadClusterCredentials(DirectoryInfo directory) + { + var path = Path.Combine(directory.FullName, "env_output.json"); + if (!File.Exists(path)) return null; + + try + { + using var document = JsonDocument.Parse(File.ReadAllText(path)); + + // two schemas in the wild: single-cluster templates put outputs at the top level, and the AWS + // multi-cluster template nests them under .clusters.value[N]. Both are handled because which one + // you get is a property of the template somebody chose, not of anything a test can control. + var root = document.RootElement; + if (root.TryGetProperty("clusters", out var clusters) + && clusters.TryGetProperty("value", out var values) + && values.ValueKind == JsonValueKind.Array + && values.GetArrayLength() > 0) + { + root = values[0]; + } + + var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); + var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); + var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + + return name is null || user is null || password is null ? null : new ClusterCredentials(name, user, password); + } + catch (Exception) + { + // a malformed env_output.json is not fatal here: only the tests that need REST enrichment care, + // and they report it themselves rather than failing every test in the suite at discovery time + return null; + } + } + + /// + /// Reads a property that may be a bare value or a terraform-style { "value": ... } wrapper. + /// + private static string? ReadValue(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var value)) return null; + if (value.ValueKind == JsonValueKind.Object && value.TryGetProperty("value", out var inner)) value = inner; + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + public sealed record ClusterCredentials(string ClusterName, string Username, string Password) + { + public Uri RestUrl => new($"https://{ClusterName}:9443"); + + /// Never let the password reach a log; the endpoints are real and reachable. + public override string ToString() => $"{Username}@{ClusterName}"; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs new file mode 100644 index 000000000..6406b8b88 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Provisions one database shape for the tests that need it, and takes it away afterwards. +/// +/// +/// Per *shape*, not per test: creating a database on a real cluster is slow, so several test classes share one. +/// +/// The three-state behaviour matters more than the provisioning. No environment configured, or the +/// E2E_SCENARIO_TESTS gate not set, means every test skips - which is the ordinary case for anybody +/// running the full traversal, and these tests create and delete real databases. But an environment that *is* +/// configured and meant, and then does not work, must **fail**: a suite that skips on a broken environment +/// reports success for tests that never ran, and you will trust it exactly when you should not. So the absent +/// case skips from the test body, and the broken case throws from here. +/// +/// +public abstract class FaultInjectorFixture(DatabaseShape shape) : IAsyncLifetime +{ + /// + /// Shared by every database this suite creates, in every run. + /// + /// + /// The sweep that cleans up leaks from earlier runs matches on this and nothing else, so it can never touch + /// a database somebody created by hand. Worth being strict about: the alternative is a test suite that + /// deletes production-shaped things on a shared cluster. + /// + public const string NamePrefix = "sertest-"; + + private static readonly string RunId = Guid.NewGuid().ToString("n")[..6]; + + private FaultInjectorClient? _injector; + private string? _skipReason; + + public DatabaseShape Shape { get; } = shape; + + /// The database created for this fixture, once has run. + public ProvisionedDatabase? Database { get; private set; } + + public FaultInjectorEnvironment Environment => + FaultInjectorEnvironment.Current ?? throw new InvalidOperationException("no environment; call RequireAvailable first"); + + public FaultInjectorClient Injector => + _injector ?? throw new InvalidOperationException("no injector; call RequireAvailable first"); + + /// + /// Skips the calling test when there is no environment to talk to. + /// + public void RequireAvailable() + { + if (_skipReason is not null) Assert.Skip(_skipReason); + } + + public async ValueTask InitializeAsync() + { + if (FaultInjectorEnvironment.Current is null) + { + _skipReason = FaultInjectorEnvironment.UnavailableReason ?? "no fault-injector environment configured"; + return; + } + + if (!FaultInjectorEnvironment.IsEnabled) + { + _skipReason = "set E2E_SCENARIO_TESTS=true to run against a real deployment (these tests create and delete databases)"; + return; + } + + // configured and meant: from here on, problems are failures + _injector = new FaultInjectorClient(FaultInjectorEnvironment.Current.InjectorUrl); + await SweepOrphansAsync(); + Database = await CreateDatabaseAsync(); + } + + public async ValueTask DisposeAsync() + { + // unconditionally, and before anything else can throw: a database left behind holds a port and a slice + // of cluster memory, and the next run collides with it + if (Database is { } database && _injector is { } injector) + { + try + { + await injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = database.BdbId }); + } + catch (Exception ex) + { + // never mask a test failure with a cleanup failure; the sweep will get it next time + Console.WriteLine($"failed to delete {database.Name} (bdb {database.BdbId}): {ex.Message}"); + } + } + + _injector?.Dispose(); + } + + /// + /// Creates the database for this shape, working around port collisions the way go-redis has to. + /// + /// + /// Port collisions are common enough on a shared cluster that go-redis carries a dedicated + /// CreateDatabaseWithPortRetry helper; this is the same idea. The base port is deliberately high and + /// the walk is upward, so a collision costs one attempt rather than a failed run. + /// + private async Task CreateDatabaseAsync() + { + const int BasePort = 13500, Attempts = 8; + var name = $"{NamePrefix}{Shape.Label}-{RunId}"; + Exception? last = null; + + for (int attempt = 0; attempt < Attempts; attempt++) + { + var port = BasePort + (attempt * 3); + try + { + // nested under database_config, not flattened: the injector answers a flat payload with + // "Invalid parameter 'database_config': got None" + var result = await Injector.RunActionAsync( + "create_database", + new Dictionary { ["database_config"] = Shape.ToCreateParameters(name, port) }); + return ProvisionedDatabase.FromCreateResult(name, port, Shape, result, Environment); + } + catch (Exception ex) when (IsPortCollision(ex)) + { + last = ex; + Console.WriteLine($"create_database on port {port} collided, retrying higher"); + } + } + + throw new InvalidOperationException($"could not create '{name}' after {Attempts} attempts", last); + } + + /// + /// Whether a create failure is worth trying a different port for. + /// + /// + /// Only port collisions are: everything else - a malformed config, a cluster with no capacity - fails + /// identically on every port, and retrying eight times turns one clear error message into eight and delays + /// the report by half a minute. The first version of this retried everything, and buried + /// "missing shard_key_regex" under eight identical tracebacks. + /// + private static bool IsPortCollision(Exception ex) => + ex.Message.Contains("port", StringComparison.OrdinalIgnoreCase) + && (ex.Message.Contains("in use", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("already", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("taken", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("conflict", StringComparison.OrdinalIgnoreCase)); + + /// + /// Best-effort removal of databases left behind by earlier runs. + /// + /// + /// Best-effort on purpose: listing databases needs the cluster REST API, which needs credentials this + /// directory may not carry, and being unable to tidy up is not a reason to fail a run that could otherwise + /// proceed. The per-fixture delete in is the reliable path; this only catches + /// leaks from a run that was killed. + /// + private async Task SweepOrphansAsync() + { + var cluster = Environment.Cluster; + if (cluster is null) + { + Console.WriteLine("no cluster credentials in env_output.json; skipping orphan sweep"); + return; + } + + try + { + using var rest = new ClusterRestClient(cluster, Environment.CertificateAuthorityPath); + foreach (var (bdbId, name) in await rest.ListDatabasesAsync()) + { + if (!name.StartsWith(NamePrefix, StringComparison.Ordinal)) continue; // never anything but ours + + Console.WriteLine($"sweeping orphaned test database {name} (bdb {bdbId})"); + await Injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = bdbId }); + } + } + catch (Exception ex) + { + Console.WriteLine($"orphan sweep skipped: {ex.Message}"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs new file mode 100644 index 000000000..a6268eb87 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A database this suite created, and how to connect to it. +/// +/// +/// Note what this type does *not* need to do: read endpoints.json to discover whether the database is +/// oss_cluster, or what endpoint type it advertises. A test that provisioned the database knows what it +/// asked for, which is the real argument for provisioning from inside the suite - the alternative is carrying +/// those facts from whoever set the environment up into the run, by hand. +/// +public sealed class ProvisionedDatabase +{ + private ProvisionedDatabase(string name, int bdbId, string host, int port, DatabaseShape shape, FaultInjectorEnvironment environment) + { + Name = name; + BdbId = bdbId; + Host = host; + Port = port; + Shape = shape; + Environment = environment; + } + + public string Name { get; } + + public int BdbId { get; } + + public string Host { get; } + + public int Port { get; } + + public DatabaseShape Shape { get; } + + public FaultInjectorEnvironment Environment { get; } + + public string? Password { get; private init; } + + /// + /// Connection options for this database, with maintenance notifications requested. + /// + /// + /// rather than Auto: in a test, a server that + /// silently declines the opt-in should fail the connection loudly rather than produce a run that passes + /// while observing nothing. That is the opposite of the right default for production, and exactly right + /// here. + /// + public ConfigurationOptions GetClientConfig() + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = MaintenanceNotificationMode.Enabled, + AbortOnConnectFail = false, + // real network, real cluster: connect and command budgets have to tolerate a WAN round trip, and a + // cluster that is mid-scenario is slower still + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Shape.Tls) + { + options.Ssl = true; + options.SslHost = Host; + + // The certificates are self-signed per environment, so trusting the issuer is what makes a TLS test + // mean anything. If the CA is missing we fail here rather than disabling validation: a TLS test that + // quietly stops checking identity reports success for the one thing it exists to catch. + var caPath = Environment.CertificateAuthorityPath + ?? throw new InvalidOperationException( + $"{Shape.Label} needs TLS, but no CA certificate was found in {Environment.ConfigDirectory.FullName}; " + + "validation is not disabled for tests"); + options.TrustIssuer(caPath); + } + + return options; + } + + /// Masked, because these endpoints are real and reachable from the internet. + public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, {Shape.Label})"; + + /// + /// Reads what the injector reported back after create_database. + /// + /// + /// Tolerant by design: the response shape is documented as prose, and the fields worth having may arrive at + /// the top level or nested under an output/result object. What cannot be guessed is the bdb_id - + /// without it there is nothing to delete afterwards - so that one is required and its absence is loud. + /// + public static ProvisionedDatabase FromCreateResult( + string name, + int requestedPort, + DatabaseShape shape, + JsonElement result, + FaultInjectorEnvironment environment) + { + var bdbId = FindInt(result, "bdb_id") + ?? throw new InvalidOperationException($"create_database for '{name}' returned no bdb_id: {result}"); + + var host = FindString(result, "endpoint") ?? FindString(result, "dns_name") ?? environment.Cluster?.ClusterName + ?? throw new InvalidOperationException($"create_database for '{name}' returned no endpoint, and env_output.json names no cluster: {result}"); + + // an endpoint may arrive as "host:port" or as a bare host + var port = requestedPort; + var colon = host.LastIndexOf(':'); + if (colon > 0 && int.TryParse(host[(colon + 1)..], out var parsedPort)) + { + port = parsedPort; + host = host[..colon]; + } + + return new ProvisionedDatabase(name, bdbId, host, port, shape, environment) + { + Password = FindString(result, "password"), + }; + } + + private static int? FindInt(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.Number && candidate.TryGetInt32(out var value)) return value; + if (candidate.ValueKind == JsonValueKind.String && int.TryParse(candidate.GetString(), out var parsed)) return parsed; + } + + return null; + } + + private static string? FindString(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.String) return candidate.GetString(); + if (candidate.ValueKind == JsonValueKind.Array && candidate.GetArrayLength() > 0 && candidate[0].ValueKind == JsonValueKind.String) + { + return candidate[0].GetString(); // endpoints arrive as a list often enough + } + } + + return null; + } + + /// + /// Yields every value for a property name, at any depth - shallowest first. + /// + private static IEnumerable Walk(JsonElement element, string name) + { + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty(name, out var direct)) yield return direct; + + foreach (var property in element.EnumerateObject()) + { + foreach (var nested in Walk(property.Value, name)) yield return nested; + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + foreach (var nested in Walk(item, name)) yield return nested; + } + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs new file mode 100644 index 000000000..3bbc96e30 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs @@ -0,0 +1,204 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The fault injector's HTTP surface, as much of it as this suite needs. +/// +/// +/// Deliberately the same shape go-redis, redis-py and node-redis wrap: POST /action returning an id and +/// GET /action/{id} to poll. They integrated independently and converged, so the contract is stable +/// enough to depend on, and their scenario documentation transfers to this client. +/// +public sealed class FaultInjectorClient(Uri baseAddress) : IDisposable +{ + private readonly HttpClient _http = new() { BaseAddress = baseAddress, Timeout = TimeSpan.FromMinutes(2) }; + + /// + /// Statuses that mean "still going". Both of them, which is the point. + /// + /// + /// A job passes through pending *and* running, so a loop that waits only on pending + /// returns while the work is still in flight - and then the test asserts against a cluster that has not + /// finished changing. Documented as a trap in the console's notes; encoded here so it cannot be + /// rediscovered. + /// + private static readonly HashSet PendingStatuses = new(StringComparer.OrdinalIgnoreCase) { "pending", "running", "in_progress" }; + + private static readonly HashSet FailedStatuses = new(StringComparer.OrdinalIgnoreCase) { "failed", "cancelled", "error" }; + + public void Dispose() => _http.Dispose(); + + /// + /// Fires an action and returns its id, without waiting. + /// + public async Task StartActionAsync(string type, object? parameters = null, CancellationToken cancellationToken = default) + { + var payload = new ActionRequest(type, parameters); + using var response = await _http.PostAsJsonAsync("/action", payload, cancellationToken); + await EnsureSuccessAsync(response, $"POST /action ({type})", cancellationToken); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + return ReadActionId(document.RootElement) + ?? throw new InvalidOperationException($"the injector accepted '{type}' but returned no action id: {document.RootElement}"); + } + + /// + /// Fires an action and waits for it to finish, returning its final payload. + /// + public async Task RunActionAsync( + string type, + object? parameters = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var id = await StartActionAsync(type, parameters, cancellationToken); + return await WaitForActionAsync(id, timeout, cancellationToken); + } + + /// + /// Polls an action to completion. + /// + /// + /// The timeout is generous by default because these actions move data: a slot migration or a node coming + /// out of maintenance mode is minutes, not seconds. A tight default here would produce failures that look + /// like product bugs. + /// + public async Task WaitForActionAsync( + string id, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromMinutes(10)); + string? lastStatus = null; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + using var response = await _http.GetAsync($"/action/{id}", cancellationToken); + await EnsureSuccessAsync(response, $"GET /action/{id}", cancellationToken); + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(body); + var status = ReadStatus(document.RootElement); + lastStatus = status ?? lastStatus; + + if (status is not null && !PendingStatuses.Contains(status)) + { + if (FailedStatuses.Contains(status)) + { + throw new InvalidOperationException($"fault-injector action {id} ended as '{status}': {body}"); + } + + // clone: the JsonDocument is disposed with this scope, and callers keep the result + return document.RootElement.Clone(); + } + + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"fault-injector action {id} was still '{lastStatus ?? "unknown"}' after {(timeout ?? TimeSpan.FromMinutes(10)).TotalSeconds:0}s"); + } + + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + } + + /// + /// Discovers which triggers are valid for an effect. + /// + /// + /// Worth asking rather than hardcoding: the effect/trigger matrix is sparse - maintenance_mode only + /// supports remove-add and remove, failover needs replication enabled - and some + /// scenarios do not enumerate their triggers in the schema at all. + /// + public async Task GetValidTriggersAsync(string scenario, string effect, int clusterIndex = 0, CancellationToken cancellationToken = default) + { + using var response = await _http.GetAsync($"/{scenario}?effect={Uri.EscapeDataString(effect)}&cluster_index={clusterIndex}", cancellationToken); + await EnsureSuccessAsync(response, $"GET /{scenario}?effect={effect}", cancellationToken); + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + return document.RootElement.Clone(); + } + + /// + /// Runs a scenario's setup / run / teardown triple's individual legs. + /// + public async Task PostScenarioAsync( + string scenario, + string? leg, + IReadOnlyDictionary query, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var path = leg is null ? $"/{scenario}" : $"/{scenario}/{leg}"; + var separator = '?'; + foreach (var pair in query) + { + if (pair.Value is null) continue; + path += $"{separator}{pair.Key}={Uri.EscapeDataString(pair.Value)}"; + separator = '&'; + } + + using var response = await _http.PostAsync(path, content: null, cancellationToken); + await EnsureSuccessAsync(response, $"POST {path}", cancellationToken); + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(body); + + // the scenario legs only *enqueue*; a caller that returns here is racing the work it asked for + var id = ReadActionId(document.RootElement); + return id is null ? document.RootElement.Clone() : await WaitForActionAsync(id, timeout, cancellationToken); + } + + private static async Task EnsureSuccessAsync(HttpResponseMessage response, string what, CancellationToken cancellationToken) + { + if (response.IsSuccessStatusCode) return; + + // include the body: the injector explains refusals there, and "400 Bad Request" alone has cost people + // hours on the effect/trigger matrix + var body = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"{what} failed: {(int)response.StatusCode} {response.ReasonPhrase}. {body}"); + } + + /// + /// The id of a pollable action, if this response describes one. + /// + /// + /// Deliberately not setup_id: a scenario's setup returns a *handle* to state held in the injector, + /// which is passed back on the run and teardown legs, and is not something /action/{id} knows about. + /// Treating it as an action id polls a URL that does not exist. + /// + private static string? ReadActionId(JsonElement element) + { + foreach (var name in new[] { "action_id", "id" }) + { + if (element.TryGetProperty(name, out var value)) + { + return value.ValueKind switch + { + JsonValueKind.String => value.GetString(), + JsonValueKind.Number => value.ToString(), + _ => null, + }; + } + } + + return null; + } + + private static string? ReadStatus(JsonElement element) + => element.TryGetProperty("status", out var status) && status.ValueKind == JsonValueKind.String + ? status.GetString() + : null; + + private sealed record ActionRequest( + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("parameters")] object? Parameters); +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs new file mode 100644 index 000000000..6c90a2f6b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// One run of a fault-injector scenario: setup, fire, tear down. +/// +/// +/// The setup leg provisions its own database, which is the important part: every trigger publishes the +/// dbconfig it needs (GET /topology-change-standalone?effect=...), and all of them want +/// proxy_policy: single - a shape the environment templates do not create. So a scenario cannot be +/// pointed at an existing database, and there is no need to hand-roll create_database for one either. +/// +/// Teardown is the whole reason this is a type rather than three calls. A scenario left set up holds cluster +/// state - the flags it enabled, the database it made, nodes it excluded - and poisons every run after it. +/// +/// +public sealed class ScenarioRun : IAsyncDisposable +{ + private readonly FaultInjectorClient _injector; + private readonly string _scenario; + private readonly Action _log; + + private ScenarioRun(FaultInjectorClient injector, string scenario, string effect, string trigger, Action log) + { + _injector = injector; + _scenario = scenario; + Effect = effect; + Trigger = trigger; + _log = log; + } + + public string Effect { get; } + + public string Trigger { get; } + + /// The setup handle, passed back on the run and teardown legs. + public string? SetupId { get; private set; } + + /// The database the setup leg created, when it says which. + public int? BdbId { get; private set; } + + /// Whatever setup returned, for a test that wants to look at more than we model. + public JsonElement SetupResult { get; private set; } + + /// The database setup created, ready to connect to. + /// + /// Setup hands back everything needed - endpoint, password, TLS - so a scenario test never has to look in + /// endpoints.json or ask the cluster REST API. Verified against a live run: the response carries + /// setup_id, bdb_id, db_name, endpoints, password, tls, + /// mtls_files and config (the proxy policy it chose to satisfy the trigger). + /// + public ScenarioDatabase? Database { get; private set; } + + /// The database a scenario provisioned for itself. + public sealed record ScenarioDatabase(string Name, int BdbId, string Host, int Port, bool Tls, string? Password, string? ProxyPolicy) + { + public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, policy {ProxyPolicy ?? "?"})"; + + public ConfigurationOptions GetClientConfig( + FaultInjectorEnvironment environment, + MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = mode, + AbortOnConnectFail = false, + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Tls) + { + options.Ssl = true; + options.SslHost = Host; + options.TrustIssuer(environment.CertificateAuthorityPath + ?? throw new InvalidOperationException($"{Name} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}")); + } + + return options; + } + } + + /// + /// Sets a scenario up, ready to fire. + /// + /// + /// The trigger for the *setup* leg, when it differs from the one being fired. + /// + /// + /// The two triggers are different questions, which the naming hides. The setup leg's trigger says how to + /// *provision* - /slot-migrate/setup accepts only reshard, because provisioning a cluster + /// database is all it does - while the run leg's trigger says how to cause the effect (migrate, + /// maintenance_mode, failover). Passing the run trigger to setup earns a + /// "Trigger 'migrate' is not supported by slot-migrate/setup (only 'reshard')". + /// + public static async Task SetupAsync( + FaultInjectorClient injector, + string scenario, + string effect, + string trigger, + Action log, + IReadOnlyDictionary? extra = null, + string? setupTrigger = null, + CancellationToken cancellationToken = default) + { + var run = new ScenarioRun(injector, scenario, effect, trigger, log); + var query = new Dictionary { ["effect"] = effect, ["trigger"] = setupTrigger ?? trigger }; + if (extra is not null) + { + foreach (var pair in extra) query[pair.Key] = pair.Value; + } + + log($"setup {scenario}: effect={effect} setup-trigger={setupTrigger ?? trigger} (firing '{trigger}')"); + run.SetupResult = await injector.PostScenarioAsync(scenario, "setup", query, cancellationToken: cancellationToken); + run.SetupId = FindString(run.SetupResult, "setup_id"); + run.BdbId = FindInt(run.SetupResult, "bdb_id"); + run.Database = ReadDatabase(run.SetupResult, run.BdbId); + log($"setup complete: setup_id={run.SetupId ?? "(none)"} database={run.Database?.ToString() ?? "(none)"}"); + return run; + } + + /// + /// Fires the scenario and waits for the injector to finish its work. + /// + /// + /// Note "finished" here means the injector has done what it was asked, not that the deployment has settled: + /// the notifications, the DNS change and the socket close all trail it by seconds. Callers should watch for + /// what they expect rather than assuming completion means arrival. + /// + public async Task FireAsync(CancellationToken cancellationToken = default) + { + var query = new Dictionary + { + ["effect"] = Effect, + ["trigger"] = Trigger, + ["setup_id"] = SetupId, + ["bdb_id"] = BdbId?.ToString(), + }; + + _log($"firing {_scenario}"); + var result = await _injector.PostScenarioAsync(_scenario, leg: null, query, cancellationToken: cancellationToken); + _log($"fired: {result}"); + return result; + } + + public async ValueTask DisposeAsync() + { + var query = new Dictionary + { + // both, because setup_id lives in the injector's memory and is lost if it restarts, at which point + // bdb_id is the only handle left + ["setup_id"] = SetupId, + ["bdb_id"] = BdbId?.ToString(), + ["restore_nodes"] = "true", // put back anything the scenario excluded, or the next run starts degraded + }; + + try + { + // Its own budget rather than the caller's token: teardown has to happen even when the test was + // cancelled or timed out, which is exactly when the caller's token is already dead. + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await _injector.PostScenarioAsync(_scenario, "teardown", query, cancellationToken: timeout.Token); + _log("teardown complete"); + } + catch (Exception ex) + { + // loud, but not an exception: a teardown failure must not replace the test's own verdict + _log($"TEARDOWN FAILED ({ex.Message}) - the cluster may be left with a scenario set up, and " + + $"setup_id={SetupId ?? "(none)"} bdb_id={BdbId?.ToString() ?? "(none)"} is what to clean up by hand"); + } + } + + private static ScenarioDatabase? ReadDatabase(JsonElement setup, int? bdbId) + { + if (bdbId is not { } id) return null; + + var endpoint = FindString(setup, "endpoints"); + if (endpoint is null) return null; + + var text = endpoint; + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) text = text[(scheme + 3)..]; + var colon = text.LastIndexOf(':'); + if (colon <= 0 || !int.TryParse(text[(colon + 1)..], out var port)) return null; + + return new ScenarioDatabase( + FindString(setup, "db_name") ?? $"bdb-{id}", + id, + text[..colon], + port, + setup.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, + FindString(setup, "password"), + FindString(setup, "config")); + } + + private static string? FindString(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.String) return candidate.GetString(); + if (candidate.ValueKind == JsonValueKind.Number) return candidate.ToString(); + if (candidate.ValueKind == JsonValueKind.Array && candidate.GetArrayLength() > 0 + && candidate[0].ValueKind == JsonValueKind.String) + { + return candidate[0].GetString(); // "endpoints" is a list even when there is one + } + } + + return null; + } + + private static int? FindInt(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.Number && candidate.TryGetInt32(out var value)) return value; + if (candidate.ValueKind == JsonValueKind.String && int.TryParse(candidate.GetString(), out var parsed)) return parsed; + } + + return null; + } + + /// + /// Yields every value for a property name at any depth, shallowest first. + /// + /// + /// Tolerant on purpose: the scenario responses are documented as prose and nest differently between legs, so + /// searching beats asserting a shape - and a test that cannot find setup_id still has a logged + /// payload to work from rather than a deserialization error. + /// + private static IEnumerable Walk(JsonElement element, string name) + { + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty(name, out var direct)) yield return direct; + foreach (var property in element.EnumerateObject()) + { + foreach (var nested in Walk(property.Value, name)) yield return nested; + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + foreach (var nested in Walk(item, name)) yield return nested; + } + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs new file mode 100644 index 000000000..9886ee68b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs @@ -0,0 +1,95 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Telling "this deployment cannot do that" apart from "the client got it wrong". +/// +/// +/// The effect/trigger matrix is sparse in ways that depend on the *cluster*, not just the database: an +/// add migration needs a node holding more than one shard, and a three-node cluster with sparse shard +/// placement gives every node exactly one. The injector reports that as a failed action with a Python +/// traceback, which is indistinguishable from a real fault unless you read the message. +/// +/// So these skip, narrowly and by message. A broad catch here would hide genuine injector failures, which is +/// the thing this tier exists to surface - hence matching on the specific condition rather than on the +/// exception type. +/// +/// +internal static class ScenarioSupport +{ + private static readonly string[] PlacementLimitations = + [ + "No node with multiple shards found", + "not enough nodes", + "no empty node", + ]; + + /// + /// Skips when the setup could not produce a database the effect can act on. + /// + public static void RequireEffectIsAchievable(ScenarioRun scenario, string effect) + { + if (scenario.Database is null) + { + Assert.Skip($"the injector did not provision a database for '{effect}'"); + } + } + + /// + /// Fires a scenario, skipping rather than failing when the cluster's shape cannot produce the effect. + /// + public static async Task FireOrSkipAsync(ScenarioRun scenario, string effect, CancellationToken cancellationToken) + { + try + { + await scenario.FireAsync(cancellationToken); + } + catch (Exception ex) when (IsPlacementLimitation(ex)) + { + Assert.Skip($"this cluster cannot produce '{effect}': {Summarize(ex.Message)}"); + } + } + + private static bool IsPlacementLimitation(Exception ex) + { + foreach (var limitation in PlacementLimitations) + { + if (ex.Message.Contains(limitation, StringComparison.OrdinalIgnoreCase)) return true; + } + + return false; + } + + /// + /// The part of a Python traceback that says what actually went wrong. + /// + /// + /// Note the traceback arrives inside a JSON string, so its line breaks are the two characters + /// \n rather than real newlines - splitting on '\n' alone matches nothing and returns + /// the whole wall of text, which is how the first version of this behaved. The injector helpfully ends with + /// "Caused by: ...", which is the only part worth putting in a skip message. + /// + internal static string Summarize(string message) + { + var caused = message.LastIndexOf("Caused by:", StringComparison.Ordinal); + if (caused >= 0) + { + var tail = message[caused..]; + var end = tail.IndexOf("\\n", StringComparison.Ordinal); + return end > 0 ? tail[..end] : tail; + } + + var lines = message.Replace("\\n", "\n", StringComparison.Ordinal) + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + for (int i = lines.Length - 1; i >= 0; i--) + { + if (lines[i].Contains("Exception:", StringComparison.Ordinal)) return lines[i]; + } + + return lines.Length > 0 ? lines[^1] : message; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs new file mode 100644 index 000000000..14fe17dbd --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// D6 against a real deployment: do we get off the connection before the server takes it away? +/// +/// +/// The one thing the in-process harness structurally cannot test. The handoff's hostname branch needs two +/// things a fake cannot supply - an endpoint that is a *name*, and a real socket whose remote address tells us +/// where we currently are - and its decision turns on DNS changing underneath us. +/// +/// The discriminator is the failure type of the disconnect. ConnectionDisposed means *we* replaced the +/// connection; SocketClosed means the server did. Both outcomes are legitimate, because DNS is not +/// guaranteed to win the race - measured on one cluster at +18.7s against a socket closing at +15.7s - so this +/// records which happened rather than demanding the good one. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "moving-handoff")] +public class MovingHandoffScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("conn_drop", "endpoint_rebind")] + [InlineData("data_movement_conn_drop", "maintenance_mode")] + public async Task HandoffBeatsTheServerToTheClose(string effect, string trigger) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", effect, trigger, log.WriteLine, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var clock = Stopwatch.StartNew(); + var timeline = new List(); + void Note(string what) + { + var entry = $" +{clock.Elapsed.TotalSeconds,6:0.0}s {what}"; + lock (timeline) timeline.Add(entry); + log.WriteLine(entry); + } + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var muxer = (IInternalConnectionMultiplexer)conn; + var endpoint = muxer.GetServerEndPoint(conn.GetEndPoints()[0]); + + // The precondition the fake could not meet: a name to resolve, and a socket that says where we are. + Assert.IsType(conn.GetEndPoints()[0]); + log.WriteLine($"dialled {conn.GetEndPoints()[0]} (a name, so the probe can engage)"); + + var failures = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (failures) failures.Add(e.FailureType); + Note($"disconnect: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => Note("reconnected"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) Note($"{push.NotificationType} seq={push.SequenceId} time={push.Time?.TotalSeconds.ToString() ?? "-"}"); + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + Note("injector reports the scenario finished"); + + // long enough to cover the whole announced window plus the observed overshoot + var deadline = clock.Elapsed + TimeSpan.FromSeconds(75); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + Note($"ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + Note($"handoff outcome: {endpoint.LastHandoffOutcome ?? "(none recorded)"}; recycles={endpoint.HandoffRecycles}"); + + lock (failures) + { + var order = string.Join(" -> ", failures); + log.WriteLine($" disconnect sequence: {(order.Length == 0 ? "(none)" : order)}"); + + // Note what is *not* asserted: that a ConnectionDisposed appears here. Our own recycle does not + // raise ConnectionFailed - disposal is not reported as a failure - so from the outside a handoff is + // invisible, which is worth knowing in its own right and is why HandoffRecycles exists. + } + + // Exactly one handoff per notification. More than one is the feedback loop this test found on its first + // live run: a server re-sends MOVING to a connection that opts in while the window is still open, and + // since the handoff replaces the connection, acting on the repeat produces another one - twelve + // recycles from a single event. The per-server sequence dedup now gates it, and this is the assertion + // that would catch a regression. + Assert.Equal(1, endpoint.HandoffRecycles); + Assert.NotNull(endpoint.LastHandoffOutcome); + Assert.Contains("Recycle", endpoint.LastHandoffOutcome); + + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should be serving commands after the handoff"); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs new file mode 100644 index 000000000..0d93b2db2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs @@ -0,0 +1,21 @@ +using System; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Waits for a condition that a real deployment reaches on its own schedule. +/// +internal static class Poll +{ + public static async Task UntilAsync(Func condition, int timeoutMilliseconds = 10_000, int pollMilliseconds = 250) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMilliseconds); + while (true) + { + if (condition()) return true; + if (DateTime.UtcNow > deadline) return false; + await Task.Delay(pollMilliseconds); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs new file mode 100644 index 000000000..dd2135c74 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A replicated database, provisioned by us - the only way to reach the failover family. +/// +/// +/// The environment templates create databases with replication: false, and a failover needs a replica to +/// promote, so this is the one case where create_database earns its keep rather than being a +/// less-convenient alternative to the scenario setup legs. +/// +public sealed class ReplicatedDatabaseFixture() + : FaultInjectorFixture(new DatabaseShape("replicated", ProxyPolicy: "single", Replication: true, ShardCount: 2)); + +/// +/// The proxy and failover families: FAILING_OVER/FAILED_OVER, and a proxy restarting underneath us. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "failover")] +public class FailoverScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Fact] + public async Task FailoverIsAnnouncedAndSurvived() + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + log.WriteLine($"provisioned {database}"); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + } + }; + + var db = conn.GetDatabase(); + await db.StringSetAsync("fi-failover", "before"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "failover", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + // The failover action's parameters are untyped in the schema, so a rejection here is a harness + // problem rather than a client finding; say so plainly instead of reporting it as a product failure. + Assert.Skip($"the injector would not run 'failover' against bdb {database.BdbId}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports the failover finished"); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + try + { + await db.PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s): {string.Join(", ", events.Select(e => e.NotificationType))}"); + + // A failover is the one event whose *pair* we have never observed end to end from a client: Marc + // captured the frames by hand, but nothing has watched SE.Redis receive them. + Assert.NotEmpty(events); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.FailingOver + or MaintenanceNotificationType.FailedOver + or MaintenanceNotificationType.Migrating // a failover on a proxied database moves shards too + or MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.Moving); + } + + // the data survived, which is the point of replication + Assert.Equal("before", await db.StringGetAsync("fi-failover")); + } +} + +/// +/// The proxy process restarting: no topology change, no notification, just the socket going away. +/// +/// +/// Included because it is the one disruption that is *purely* a connection event - nothing moves, nothing is +/// announced, and recovery is entirely the client's ordinary reconnect path. A useful control: if this fails, +/// the failures in the announced scenarios are not about notifications at all. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "proxy-restart")] +public class ProxyRestartScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("standalone")] + [InlineData("cluster")] + public async Task ProxyRestartIsSurvived(string key) + { + var database = fixture.Require(key); + var cancellationToken = TestContext.Current.CancellationToken; + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var db = conn.GetDatabase(); + await db.StringSetAsync($"fi-dmc-{key}", "before"); + + int drops = 0; + conn.ConnectionFailed += (_, e) => + { + drops++; + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s connection failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s connection restored"); + + clock.Restart(); + // the one action with a typed parameter object in the injector's schema: RestartDmcParams { bdb_id } + await fixture.Injector.RunActionAsync( + "dmc_restart", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s DMC restart reported finished"); + + // Recovery is asserted by polling rather than by waiting a fixed time, because a proxy restart is quick + // and the interesting failure is "never comes back", not "takes a moment". + Assert.True( + await Poll.UntilAsync(() => TryRead(db, $"fi-dmc-{key}"), timeoutMilliseconds: 60_000), + "the client should recover from the proxy restarting"); + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered after {drops} drop(s)"); + Assert.Equal("before", await db.StringGetAsync($"fi-dmc-{key}")); + } + + private static bool TryRead(IDatabase db, string key) + { + try + { + return db.StringGet(key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md new file mode 100644 index 000000000..2be42b3a7 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -0,0 +1,60 @@ +# StackExchange.Redis.FaultInjector.Tests + +Scenario tests that drive a **real Redis Enterprise deployment** through the fault injector, and watch how +SE.Redis reacts. These are the tier that can observe what no in-process fake can: real DNS, real TLS identity, +real timing. + +## Running them + +```bash +cd # holds docker-compose.yml, env_output.json, ... +docker compose up -d # 10-15 minutes for the cluster to come up + +export SER_FI_CONFIG_DIR=$PWD # or FI_CONSOLE_CONFIG_DIR, which is also honoured +export E2E_SCENARIO_TESTS=true # explicit opt-in: these create and delete databases +dotnet test tests/StackExchange.Redis.FaultInjector.Tests +``` + +One path is the whole configuration. That directory is the one mounted into the injector as `/app/config`, so +it already holds the cluster credentials (`env_output.json`), the CA certificate, and the compose file; nothing +has to be hand-carried into the test run. `FAULT_INJECTION_API_URL` overrides the injector URL +(default `http://127.0.0.1:20324`). + +## Three states, deliberately distinct + +| state | behaviour | +|---|---| +| no `SER_FI_CONFIG_DIR` | every test **skips** - the ordinary case, including `build.ps1`'s full traversal | +| directory set, no `E2E_SCENARIO_TESTS=true` | every test **skips** - nobody should create databases by accident | +| configured and enabled, but broken | every test **fails** | + +The third row is the important one. A suite that skips when the environment is broken reports success for tests +that never ran, and it will be trusted at exactly the wrong moment. + +## Databases are created by the tests, not by you + +Each *shape* (`DatabaseShape`) is a fixture shared by the classes that need it, because creating a database on a +real cluster is slow. The shapes exist because they change client behaviour rather than for coverage's sake: the +number of A records a hostname carries follows proxy placement, and the handoff takes a different branch +depending on whether a live sibling address exists. + +Every database is named `sertest--`. Cleanup is per fixture and unconditional; a sweep at startup +removes leaks from runs that were killed, matching on the `sertest-` prefix and nothing else, so it can never +touch a database created by hand. + +## TLS + +Certificates are self-signed per environment, so tests call `ConfigurationOptions.TrustIssuer(caPath)` with the +CA found in the config directory. If the CA is missing, TLS tests **fail** rather than disabling validation - a +TLS test that quietly stops checking identity reports success for the one thing it exists to catch. + +## Traits + +`tier=fault-injector` on everything, so the whole tier can be excluded in one filter; `scenario=` for +subsets. + +## Unverified + +The `create_database` parameter names in `DatabaseShape.ToCreateParameters` are the injector's wire schema, +which is documented only as prose. They are gathered in one place so a real run can correct them; go-redis's +`DatabaseConfig` is the closest reference implementation. Treat them as unconfirmed until a run accepts them. diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs new file mode 100644 index 000000000..a5f37b8b6 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Does the client work against a real deployment at all: connect, negotiate RESP3, opt in, run a command. +/// +/// +/// The floor for everything else. Worth having as its own class because when a scenario test fails, the first +/// question is whether the deployment was reachable in the first place, and a green smoke test answers it +/// without reading logs. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "smoke")] +public class RealDeploymentSmokeTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("standalone")] + [InlineData("cluster")] + public async Task OptInIsAcceptedAndTheConnectionWorks(string key) + { + var database = fixture.Require(key); + log.WriteLine($"{database}"); + log.WriteLine($" advertised addresses: {string.Join(", ", database.Addresses)}"); + log.WriteLine($" endpoint type: {database.EndpointType ?? "(unset)"}"); + + // Enabled refuses the connection if the server will not give us notifications, so reaching the + // assertions at all is the opt-in having been accepted - there is nothing weaker to check. A stub +OK + // would pass this, which is what the scenario tests are for. + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + Assert.True(conn.IsConnected); + + var server = conn.GetServer(conn.GetEndPoints()[0]); + log.WriteLine($" connected: {server.Version}, {server.ServerType}, protocol {server.Protocol}"); + Assert.Equal(RedisProtocol.Resp3, server.Protocol); + + var rtt = await conn.GetDatabase().PingAsync(); + log.WriteLine($" ping: {rtt.TotalMilliseconds:0.0}ms"); + Assert.True(rtt > TimeSpan.Zero); + + // and the feature is actually live on this connection, not merely requested + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "maintenance notifications should be active"); + } + + [Fact] + public async Task AdvertisedAddressCountMatchesTheProxyPolicy() + { + // Not a client assertion: a check that the environment is the shape the handoff tests assume. The count + // follows proxy *placement* rather than the policy name, so this records what this deployment actually + // is - and a handoff test that expects a sibling to step to needs more than one. + var standalone = fixture.Require("standalone"); + log.WriteLine($"{standalone.Key}: policy {standalone.ProxyPolicy}, {standalone.AdvertisedAddressCount} address(es)"); + + Assert.NotEmpty(standalone.Addresses); + if (standalone.ProxyPolicy is "single") + { + Assert.Equal(1, standalone.AdvertisedAddressCount); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj new file mode 100644 index 000000000..052d85e11 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + Exe + StackExchange.Redis.FaultInjector.Tests + true + true + full + enable + true + + + + + + + + + + + + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs new file mode 100644 index 000000000..a99be3bc2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The same feature over TLS, against certificates a real cluster generated. +/// +/// +/// A gap that has been open since the identity work: nothing had exercised a real TLS deployment, only the +/// in-process harness with a certificate it made itself. That matters because the interesting failures are +/// about *identity* - which name the certificate carries, and whether the endpoint we are told to use is +/// covered by it - and a fake that issues its own certificate cannot be wrong about that in the way a real +/// deployment can. +/// +/// Certificates here are self-signed per environment, so the client pins the issuer with +/// . Note what is deliberately *not* done: validation is +/// never disabled. A TLS test that stops checking identity reports success for precisely the thing it exists to +/// catch, so a missing CA fails the run instead. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "tls")] +public class TlsScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Fact] + public async Task NotificationsArriveOverTlsAndIdentityIsVerified() + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + if (fixture.Environment.CertificateAuthorityPath is null) + { + // a skip rather than a failure: whether the environment generated certificates is a provisioning + // choice, and running this without them would prove nothing + Assert.Skip($"no CA certificate in {fixture.Environment.ConfigDirectory.FullName}; provision with TLS to run this"); + } + + log.WriteLine($"trusting issuer {fixture.Environment.CertificateAuthorityPath}"); + CertificateSanity.RequireCertificatesMatchThisCluster(fixture.Environment, log.WriteLine); + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, + "topology-change-standalone", + "conn_drop", + "endpoint_rebind", + log.WriteLine, + // include_tls does not *request* TLS, it widens the list of variants the setup may choose from; + // variant_index is what picks one. Confirmed by asking the discovery endpoint: with no flags a + // trigger offers one variant ("single"), with include_tls two ("single", "single_tls"), and with + // include_mtls a third ("mtls"). Passing include_tls alone provisions variant 0 and yields a + // plaintext database, which is how this test first came to skip itself. + extra: new Dictionary + { + ["include_tls"] = "true", + ["variant_index"] = "1", + }, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + log.WriteLine($"provisioned {database} (tls={database.Tls})"); + + if (!database.Tls) + { + Assert.Skip("the injector provisioned a plaintext database despite include_tls=true; nothing to test here"); + } + + var clock = Stopwatch.StartNew(); + var events = new List(); + + var config = database.GetClientConfig(fixture.Environment); + + // AbortOnConnectFail=true *for this test only*: everywhere else tolerating a slow start is right, but a + // TLS failure has to surface its reason. With it false, a certificate problem is indistinguishable from + // a slow cluster - ConnectAsync succeeds, IsConnected is false, and the cause is gone. + config.AbortOnConnectFail = true; + + // If the certificate does not cover the name we dialled, this throws - which is the point: the + // assertion that identity was verified is the connect succeeding with validation on. The log goes to + // test output so a handshake failure says *which* check failed. + var connectLog = new StringWriter(); + ConnectionMultiplexer conn; + try + { + conn = await ConnectionMultiplexer.ConnectAsync(config, connectLog); + } + catch (Exception ex) + { + log.WriteLine(connectLog.ToString()); + log.WriteLine($"TLS connect failed: {ex.GetType().Name}: {ex.Message}"); + throw; + } + + await using (conn) + { + Assert.True(conn.IsConnected); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "the opt-in should be live over TLS too"); + log.WriteLine($"connected over TLS; opt-in active; ping {(await conn.GetDatabase().PingAsync()).TotalMilliseconds:0.0}ms"); + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + } + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s) over TLS"); + Assert.NotEmpty(events); + } + + // and the TLS handshake succeeds again on the *replacement* connection, which is the part a + // certificate-name problem would break rather than the first connect + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should re-establish TLS after the endpoint moves"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs new file mode 100644 index 000000000..4c79f1e01 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Real topology changes on a real deployment, watched by a real client. +/// +/// +/// Each scenario provisions its own database, because every trigger publishes the dbconfig it requires +/// and all of them want proxy_policy: single - a shape the environment templates do not create. +/// +/// What these assert is deliberately modest: that the notifications arrive, parse, and open the relaxation +/// window. Timings are *recorded* rather than asserted, because the measured spread across clusters is wide +/// enough that a bound tight enough to be interesting would be flaky - DNS has been seen updating 4.4s after +/// MOVING on one cluster and 3s after the socket closed on another. The log is the deliverable for +/// timing; the assertions cover behaviour. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "topology-change")] +public class TopologyChangeScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + /// + /// Records what arrived and when, relative to the moment the scenario was fired. + /// + private sealed class Timeline(Stopwatch clock, ITestOutputHelper log) + { + private readonly List _entries = []; + + public List Events { get; } = []; + + public void Note(string what) + { + var entry = $" +{clock.Elapsed.TotalSeconds,6:0.0}s {what}"; + lock (_entries) + { + _entries.Add(entry); + } + + log.WriteLine(entry); + } + + public void Add(PushMaintenanceEvent evt) + { + lock (_entries) + { + Events.Add(evt); + } + + Note($"{evt.NotificationType} seq={evt.SequenceId} time={evt.Time?.TotalSeconds.ToString() ?? "-"} {evt.RawMessage}"); + } + + public int Count + { + get { lock (_entries) { return Events.Count; } } + } + } + + /// + /// Whether each scenario announces itself, and why - measured 2026-09-01 against RS 8.0.22. + /// + /// + /// The expectations encode the rule the measurements produced, which is narrower than either half of it + /// looks: a connection is told MOVING when its own proxy leaves the endpoint's address set *and* + /// the set gains a member. Both conditions are needed - a pure reduction takes the proxy away without + /// announcing it, and a pure widening adds addresses while leaving the connection's proxy in place, which is + /// also silent. The data-movement pair (MIGRATING/MIGRATED) is separate and fires whenever + /// shards move, whether or not any endpoint changes. + /// + [Theory] + [InlineData("conn_drop", "endpoint_rebind", true)] + [InlineData("dns_resolution_change", "endpoint_rebind", false)] + [InlineData("data_movement_conn_drop", "maintenance_mode", true)] + [InlineData("data_movement_no_conn_drop", "migrate", true)] + public async Task ScenarioProducesNotificationsWeUnderstand(string effect, string trigger, bool expectNotifications) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", effect, trigger, log.WriteLine, cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var clock = Stopwatch.StartNew(); + var timeline = new Timeline(clock, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "the opt-in must be live, or this test proves nothing"); + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) timeline.Add(push); + }; + conn.ConnectionFailed += (_, e) => timeline.Note($"connection failed: {e.FailureType} {e.Exception?.Message}"); + conn.ConnectionRestored += (_, e) => timeline.Note("connection restored"); + + clock.Restart(); + timeline.Note($"firing {effect}/{trigger} against {database}"); + await scenario.FireAsync(cancellationToken); + timeline.Note("injector reports the scenario finished"); + + // The injector finishing is not the deployment settling: notifications, the DNS change and the socket + // close all trail it. Keep watching, and keep the client busy so a broken connection actually surfaces + // rather than sitting idle. + var deadline = clock.Elapsed + TimeSpan.FromSeconds(60); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + timeline.Note($"ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + timeline.Note($"finished with {timeline.Count} notification(s); relaxed={endpoint.IsMaintenanceRelaxed}"); + + if (expectNotifications) + { + Assert.NotEmpty(timeline.Events); + Assert.All(timeline.Events, e => Assert.NotEqual(MaintenanceNotificationType.None, e.NotificationType)); + } + else + { + // dns_resolution_change widens the policy (single -> all-master-shards), so addresses are *added* + // and the connection's own proxy stays where it is - there is nothing to tell this connection to + // move, and the proxy restarting to apply the change closes the socket with no warning at all. + // Asserting the silence deliberately: it is the case with no signal, so if a future build starts + // announcing it, that is a behaviour change we want to be told about rather than to absorb quietly. + Assert.Empty(timeline.Events); + } + + // and the client is usable afterwards, which is the point of the whole feature + Assert.True( + await Poll.UntilAsync(() => TryPing(conn), timeoutMilliseconds: 30_000), + "the client should be serving commands again after the topology change"); + } + + private static bool TryPing(IConnectionMultiplexer conn) + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + } +} diff --git a/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs b/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs new file mode 100644 index 000000000..302c880f1 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs @@ -0,0 +1,216 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The DNS probe behind the MOVING handoff. Tested against an injected resolver rather than real DNS, +/// which is the only way to exercise the case that actually happens: the first answer naming the address we +/// were just told to leave. +/// +public class AdvertisedAddressProbeTests(ITestOutputHelper log) +{ + private static readonly IPAddress Retiring = IPAddress.Parse("10.129.228.140"); + private static readonly IPAddress Replacement = IPAddress.Parse("10.252.90.18"); + private static readonly DnsEndPoint Endpoint = new("db.example.cloud.redislabs.com", 13486); + + /// + /// Resolves from a script: one entry per call, the last repeating forever. + /// + private sealed class ScriptedResolver(params IPAddress[][] answers) + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + + public Task ResolveAsync(string host, CancellationToken cancellationToken) + { + var index = Interlocked.Increment(ref _calls) - 1; + return Task.FromResult(answers[Math.Min(index, answers.Length - 1)]); + } + } + + [Fact] + public async Task DnsTrailingTheNotificationIsPolledThrough() + { + // The measured case: DNS named the retiring address for the first 4.4-9.7 seconds after MOVING. A + // client that accepted the first answer would hand off to the node it was told to leave. + var resolver = new ScriptedResolver( + [Retiring], + [Retiring], + [Retiring], + [Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(4, resolver.Calls); + } + + [Fact] + public async Task ReplacementOnTheFirstAnswerIsTakenImmediately() + { + var resolver = new ScriptedResolver([Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Fact] + public async Task WindowExpiringWithoutAMoveGivesUpRatherThanGuessing() + { + // Not a failure to report: the server closes the socket regardless, and the relaxed window covers the + // reconnect. Guessing an address here would be worse than doing nothing. + var resolver = new ScriptedResolver([Retiring]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromMilliseconds(120), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Null(result); + Assert.True(resolver.Calls > 1, $"should have retried within the window, but resolved {resolver.Calls} time(s)"); + } + + [Fact] + public async Task ZeroWindowStillGetsOneAttempt() + { + // "act now" rather than "do nothing": the notifications legitimately carry zero or negative times for + // a connection that arrived mid-window + var resolver = new ScriptedResolver([Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.Zero, pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Fact] + public async Task ResolutionFailureIsRetriedNotFatal() + { + // a DNS blip mid-handoff is when we can least afford to give up + int calls = 0; + Task Resolve(string host, CancellationToken cancellationToken) + { + calls++; + return calls switch + { + 1 => throw new System.Net.Sockets.SocketException(11001), // host not found + 2 => Task.FromResult([Retiring]), + _ => Task.FromResult([Replacement]), + }; + } + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolve, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(3, calls); + } + + [Fact] + public async Task MultipleAddressesTakeTheOneThatIsNotRetiring() + { + // a round-robin record can name both nodes at once mid-move + var resolver = new ScriptedResolver([Retiring, Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + } + + [Fact] + public async Task PlacementNotPolicyDecidesWhetherWeWait() + { + // The A-record count follows actual proxy placement, not the policy name: an all-master-shards + // database whose shards share a node resolves to one address, and then there is no sibling to step to + // and the wait is the only option. Same code path as `single`, which is the point - nothing here reads + // the policy. + var resolver = new ScriptedResolver([Retiring], [Retiring], [Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(3, resolver.Calls); // it waited, because there was nothing else advertised + } + + [Fact] + public async Task SiblingIsTakenWithoutWaitingForTheRecordToMove() + { + // The common case: several A records, so the first resolution already names a live sibling proxy while + // the retiring address is *still* advertised. Stepping sideways immediately is correct - any proxy of + // the same database serves the same data - and it means the poll usually never engages. + var sibling = IPAddress.Parse("10.246.250.155"); + var resolver = new ScriptedResolver([Retiring, sibling]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(sibling, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task StillAdvertisedAnswersTheUnannouncedCase(bool present) + { + // The measured gap: a multi-proxy node taken out for maintenance announced only MIGRATING/MIGRATED, + // dropped the victim from DNS at +21.4s, and closed its socket silently at +34.7s. For thirteen + // seconds the condition was visible to anyone who asked, and no notification said so. + var sibling = IPAddress.Parse("10.246.250.155"); + var resolver = new ScriptedResolver(present ? [Retiring, sibling] : [sibling, Replacement]); + + var result = await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(present, result); + } + + [Fact] + public async Task ResolutionFailureIsNotAReasonToAbandonAConnection() + { + // null rather than false, deliberately: "cannot tell" must not become "give it up", or a DNS blip + // recycles every healthy connection at once + Task Throws(string host, CancellationToken cancellationToken) + => throw new System.Net.Sockets.SocketException(11001); + + Assert.Null(await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: Throws, log: log.WriteLine)); + + // and the same for a record that momentarily resolves to nothing at all + Assert.Null(await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: (_, _) => Task.FromResult([]), log: log.WriteLine)); + } + + [Fact] + public async Task CancellationStopsThePoll() + { + using var cts = new CancellationTokenSource(); + var resolver = new ScriptedResolver([Retiring]); + + var probe = AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromMinutes(1), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine, cancellationToken: cts.Token); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => probe); + } +} diff --git a/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs new file mode 100644 index 000000000..13e551c1f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs @@ -0,0 +1,61 @@ +using System.Linq; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The comma-and-range slot form the cluster notifications carry, e.g. 123,456,789-1000. Split out +/// because it is the one part of stage 3 that a wire capture cannot invalidate - it is the same form +/// CLUSTER NODES has always used. +/// +public class ClusterSlotMigrationUnitTests(ITestOutputHelper log) +{ + [Theory] + [InlineData("123", "123-123")] + [InlineData("123,456", "123-123,456-456")] + [InlineData("789-1000", "789-1000")] + [InlineData("123,456,789-1000", "123-123,456-456,789-1000")] + [InlineData("0-16383", "0-16383")] + [InlineData("5-5", "5-5")] + public void ParsesTheSlotForm(string input, string expected) + { + Assert.True(SlotRange.TryParseList(input, out var ranges)); + var actual = string.Join(",", ranges.Select(x => $"{x.From}-{x.To}")); + log.WriteLine($"{input} -> {actual}"); + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("123,,456", "123-123,456-456")] // empty element, skipped + [InlineData("123,", "123-123")] // trailing comma + [InlineData(",123", "123-123")] // leading comma + public void ToleratesEmptyElements(string input, string expected) + { + Assert.True(SlotRange.TryParseList(input, out var ranges)); + Assert.Equal(expected, string.Join(",", ranges.Select(x => $"{x.From}-{x.To}"))); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(",")] + [InlineData("abc")] + [InlineData("12a")] + [InlineData("-5")] // no lower bound + [InlineData("5-")] // no upper bound + [InlineData("1000-789")] // reversed: a server bug, not something to normalize silently + [InlineData("1-2-3")] + [InlineData("123,abc")] // one bad element fails the list: a partial slot set is worse than none + public void RejectsMalformedInput(string? input) + { + Assert.False(SlotRange.TryParseList(input, out var ranges)); + log.WriteLine($"'{input}' rejected, {ranges.Count} range(s)"); + } + + [Fact] + public void OutOfRangeSlotIsRejected() + { + // 16384 does not fit the short, and a checked cast would throw rather than wrap + Assert.False(SlotRange.TryParseList("99999", out _)); + } +} diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 6c62e3bb2..06876e946 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -62,6 +62,10 @@ orderby name Assert.Equal( new[] { + "_maintenanceNotifications", + "_maintenancePostEventRelaxedDuration", + "_maintenanceRelaxedTimeout", + "_maintenanceRelaxedWindowMax", "_protocol", "asyncTimeout", "backlogPolicy", @@ -891,6 +895,68 @@ public void CheckHighIntegrity(bool? assigned, bool expected, string cs) Assert.Equal(expected, parsed.HighIntegrity); } + [Theory] + [InlineData("maintRelaxedTimeout=20", 20, 60, 40)] + [InlineData("maintRelaxedWindowMax=45", 10, 45, 20)] + [InlineData("maintPostEventRelaxed=0", 10, 30, 0)] + public void MaintenanceDurationsRoundTrip(string cs, int relaxedSeconds, int capSeconds, int tailSeconds) + { + // these are in *seconds*, unlike every other timeout here, because that is the unit the cross-client + // contract names for maintRelaxedTimeout - so a documented value can be pasted between clients + var options = Parse("dummy," + cs); + Assert.Equal(TimeSpan.FromSeconds(relaxedSeconds), options.MaintenanceRelaxedTimeout); + Assert.Equal(TimeSpan.FromSeconds(capSeconds), options.MaintenanceRelaxedWindowMax); + Assert.Equal(TimeSpan.FromSeconds(tailSeconds), options.MaintenancePostEventRelaxedDuration); + + // only the explicitly-set key is serialized; the others stay defaulted + Assert.Equal("dummy," + cs, RemoveTestDefaults(options.ToString())); + + var clone = options.Clone(); + Assert.Equal(options.MaintenanceRelaxedTimeout, clone.MaintenanceRelaxedTimeout); + Assert.Equal("dummy," + cs, RemoveTestDefaults(clone.ToString())); + } + + [Fact] + public void MaintenanceDurationsDefaultRelativeToTheRelaxedTimeout() + { + // the cap and tail are multiples so that raising the relaxed timeout cannot produce a cap below it + var options = Parse("dummy,maintRelaxedTimeout=60"); + Assert.Equal(TimeSpan.FromSeconds(180), options.MaintenanceRelaxedWindowMax); + Assert.Equal(TimeSpan.FromSeconds(120), options.MaintenancePostEventRelaxedDuration); + } + + [Fact] + public void MaintenanceDurationInMillisecondsIsDiagnosed() + { + // the silent-misconfiguration case: somebody assumes milliseconds like every other timeout here and + // writes 30000, which as seconds would be an eight-hour relaxed timeout. The message names the unit + var ex = Assert.Throws(() => Parse("dummy,maintRelaxedTimeout=30000")); + Output.WriteLine(ex.Message); + Assert.Contains("seconds", ex.Message); + Assert.Contains("not in milliseconds", ex.Message); + } + + [Theory] + [InlineData(null, MaintenanceNotificationMode.Disabled, "dummy")] + [InlineData(MaintenanceNotificationMode.Disabled, MaintenanceNotificationMode.Disabled, "dummy,maintNotifications=Disabled")] + [InlineData(MaintenanceNotificationMode.Enabled, MaintenanceNotificationMode.Enabled, "dummy,maintNotifications=Enabled")] + [InlineData(MaintenanceNotificationMode.Auto, MaintenanceNotificationMode.Auto, "dummy,maintNotifications=Auto")] + public void CheckMaintenanceNotifications(MaintenanceNotificationMode? assigned, MaintenanceNotificationMode expected, string cs) + { + var options = Parse("dummy"); + if (assigned.HasValue) options.MaintenanceNotifications = assigned.Value; + + Assert.Equal(expected, options.MaintenanceNotifications); + Assert.Equal(cs, RemoveTestDefaults(options.ToString())); + + var clone = options.Clone(); + Assert.Equal(expected, clone.MaintenanceNotifications); + Assert.Equal(cs, RemoveTestDefaults(clone.ToString())); + + var parsed = Parse(cs); + Assert.Equal(expected, parsed.MaintenanceNotifications); + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -902,6 +968,12 @@ public void DefaultsProviderProtocolNotSerialized(bool clone) if (clone) options = options.Clone(); Assert.Equal(RedisProtocol.Resp3, options.Protocol); Assert.Same(provider, options.Defaults); - Assert.Equal("", options.ToString()); + + // the *values* a provider supplies are still never serialized - that is what this test is for... + Assert.DoesNotContain("protocol=", options.ToString()); + + // ...but the explicit choice of provider now is, by name, so the string describes the configuration it + // was given rather than quietly dropping part of it + Assert.Equal("defaults=amr", options.ToString()); } } diff --git a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs index fae837bd9..7da6ab839 100644 --- a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs +++ b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs @@ -86,6 +86,135 @@ public void IsMatchOnAzureManagedRedisDomain(string hostName) Assert.IsType(provider); } + [Theory] + [InlineData("amr", typeof(AzureManagedRedisOptionsProvider))] + [InlineData("AMR", typeof(AzureManagedRedisOptionsProvider))] // names are case-insensitive, like every other key + [InlineData("azure", typeof(AzureOptionsProvider))] + [InlineData("rediscloud", typeof(RedisCloudOptionsProvider))] + [InlineData("enterprise", typeof(RedisEnterpriseOptionsProvider))] + public void DefaultsProviderCanBeNamedInAConfigurationString(string name, Type expected) + { + // the on-premise case is why this exists: an Enterprise cluster has whatever DNS its operator gave it, + // so IsMatch can never recognize it, and until now the only way to select a provider was to write code + var options = ConfigurationOptions.Parse($"localhost,defaults={name}"); + Assert.IsType(expected, options.Defaults); + + // and it round-trips, because it was chosen rather than inferred + var text = options.ToString(); + Output.WriteLine(text); + Assert.Contains($"defaults={name.ToLowerInvariant()}", text); + Assert.IsType(expected, ConfigurationOptions.Parse(text).Defaults); + } + + [Fact] + public void ProviderToStringPrefersTheName() + { + // for logs: "amr" rather than a namespace-qualified type name + Assert.Equal("amr", new AzureManagedRedisOptionsProvider().ToString()); + Assert.Equal("enterprise", new RedisEnterpriseOptionsProvider().ToString()); + + // ...and an unnameable one still says something useful, which is exactly why serialization tests + // Name rather than ToString: this is never null + var custom = new TestOptionsProvider(".custom"); + Assert.Null(custom.Name); + Assert.Contains(nameof(TestOptionsProvider), custom.ToString()); + } + + [Fact] + public void UnknownDefaultsProviderNameIsRejectedWithTheAlternatives() + { + var ex = Assert.Throws(() => ConfigurationOptions.Parse("localhost,defaults=sideways")); + Output.WriteLine(ex.Message); + Assert.Contains("enterprise", ex.Message); // the message lists what it would have accepted + Assert.Contains("rediscloud", ex.Message); + } + + [Fact] + public void AnInferredDefaultsProviderIsNotSerialized() + { + // the trap this guards: the Defaults getter memoizes an inferred provider into the same field an + // explicit set writes, so without tracking *how* it got there, merely reading the property would make + // an endpoint-derived guess look like a decision - and re-parsing the string would then pin it + var options = ConfigurationOptions.Parse("contoso.cloud.redislabs.com"); + Assert.IsType(options.Defaults); // inferred, and memoized by that read + + Assert.DoesNotContain("defaults=", options.ToString()); + Assert.DoesNotContain("defaults=", options.Clone().ToString()); + } + + [Fact] + public void AnExplicitDefaultsProviderSurvivesCloning() + { + var options = ConfigurationOptions.Parse("localhost,defaults=enterprise"); + var clone = options.Clone(); + + Assert.IsType(clone.Defaults); + Assert.Contains("defaults=enterprise", clone.ToString()); + } + + [Fact] + public void AnUnnameableProviderIsNotSerializedEvenWhenExplicit() + { + // a custom provider is still perfectly usable in code; it just cannot be expressed as a string, the + // same way an inbuilt tunnel can be and a custom one cannot + var options = ConfigurationOptions.Parse("localhost"); + options.Defaults = new TestOptionsProvider(".unnameable"); + + Assert.Null(options.Defaults.Name); + Assert.DoesNotContain("defaults=", options.ToString()); + } + + [Theory] + [InlineData("redis-12345.c1.eu-west-1-2.ec2.cloud.redislabs.com")] + [InlineData("contoso.CLOUD.REDISLABS.COM")] // case-insensitive, as the sibling providers are + [InlineData("redis-12345.c1.us-east-1-2.ec2.cloud.redis.io")] // newer routing scheme + [InlineData("contoso.redislabs.com")] // older subscriptions, addressed directly + public void IsMatchOnRedisCloudDomain(string hostName) + { + var epc = new EndPointCollection(new List() { new DnsEndPoint(hostName, 0) }); + var provider = DefaultOptionsProvider.GetProvider(epc); + Assert.IsType(provider); + } + + [Fact] + public void RedisCloudDoesNotInheritTheAzureManagedAssumptions() + { + // the two deployments look similar and are not: AMR is TLS-only with a 7.4 floor, Redis Cloud is + // neither, so copying those two would break plaintext databases and over-claim the server version + var epc = new EndPointCollection(new List() { new DnsEndPoint("contoso.cloud.redislabs.com", 0) }); + var cloud = DefaultOptionsProvider.GetProvider(epc); + var amr = new AzureManagedRedisOptionsProvider(); + + Assert.True(amr.GetDefaultSsl(epc)); + Assert.False(cloud.GetDefaultSsl(epc)); + Assert.Equal(RedisFeatures.v7_4_0, amr.DefaultVersion); + Assert.Equal(DefaultOptionsProvider.BaseDefaultVersion, cloud.DefaultVersion); + + // ...but it does share the parts that are about being a proxied, hosted deployment + Assert.Equal(RedisProtocol.Resp3, cloud.Protocol); + Assert.Equal("", cloud.ConfigurationChannel); + Assert.False(cloud.AbortOnConnectFail); + } + + [Theory] + [InlineData("contoso.redis.azure.net", MaintenanceNotificationMode.Auto)] // AMR asks, pre-emptively + [InlineData("contoso.cloud.redislabs.com", MaintenanceNotificationMode.Auto)] // and Redis Cloud, which emits them + [InlineData("contoso.redis.cache.windows.net", MaintenanceNotificationMode.Disabled)] // classic Azure does not + [InlineData("contoso.example.com", MaintenanceNotificationMode.Disabled)] // and neither does anything else + public void MaintenanceNotificationDefaultPerProvider(string hostName, MaintenanceNotificationMode expected) + { + // Auto rather than Enabled is what makes the pre-emptive default safe: AMR does not emit these yet, + // so until the server side ships the opt-in is refused and the feature simply stays off + var epc = new EndPointCollection(new List() { new DnsEndPoint(hostName, 0) }); + var provider = DefaultOptionsProvider.GetProvider(epc); + Output.WriteLine($"{hostName} -> {provider.GetType().Name}"); + Assert.Equal(expected, provider.MaintenanceNotifications); + + // ...and it arrives through the options, not just off the provider + var options = new ConfigurationOptions { EndPoints = { new DnsEndPoint(hostName, 0) } }; + Assert.Equal(expected, options.MaintenanceNotifications); + } + [Theory] [InlineData(RedisProtocol.Resp2)] [InlineData(RedisProtocol.Resp3)] diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index c8a30030e..0c2d4892f 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using StackExchange.Redis.Availability; using Xunit; @@ -6,6 +6,34 @@ namespace StackExchange.Redis.Tests; public class HealthCheckPolicyUnitTests { + /// + /// Probe traffic must not be counted as work a caller is waiting on, because idleness is what decides + /// whether an endpoint that has left the deployment can be given up - a probe that looks like caller work + /// makes a server appear busy precisely because we are watching it. + /// + /// + /// The other half of this test is what it asserts is *false*: the probe must not become an internal call, + /// which would also change how it is queued - internal calls bypass the backlog. A health check that + /// bypasses the backlog cannot observe a bridge whose queue is not draining, which is exactly the signal + /// geo-redundant failover depends on. + /// + [Fact] + public void ProbeTrafficIsNotCallerFacingButIsRoutedNormally() + { + var flags = new HealthCheckContext(null!, TimeSpan.FromSeconds(1)).ProbeFlags; + Assert.NotEqual(CommandFlags.None, flags); + + var message = Message.Create(-1, flags, RedisCommand.PING); + Assert.True(message.IsProbe); + Assert.False(message.IsCallerFacing); + Assert.False(message.IsInternalCall); + + // and an ordinary command is unaffected in both directions + var ordinary = Message.Create(-1, CommandFlags.None, RedisCommand.PING); + Assert.False(ordinary.IsProbe); + Assert.True(ordinary.IsCallerFacing); + } + [Theory] [InlineData(0, 0, 5, HealthCheckResult.Inconclusive)] // No results yet [InlineData(1, 0, 0, HealthCheckResult.Healthy)] // One success, no more probes diff --git a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs index 09947fa25..7d793f18a 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs @@ -26,6 +26,32 @@ public static class TestConfig public static int MinTimeoutMilliseconds { get; } = int.TryParse(Environment.GetEnvironmentVariable("REDIS_TESTS_MIN_TIMEOUT_MS"), out var ms) && ms > 0 ? ms : 0; + /// + /// A suite-wide maintenance-notification mode, from REDIS_TESTS_MAINT_NOTIFICATIONS + /// (disabled/enabled/auto); null (the default) leaves the library's own + /// default alone. + /// + /// + /// The point is to be able to run the *whole* suite with the opt-in on, rather than to test the feature: + /// most servers we test against never send a notification, and that is the expected outcome - the opt-in + /// must be invisible to everything else. Applied only where a test has not asked for a specific mode. + /// + public static MaintenanceNotificationMode? MaintenanceNotifications { get; } = + Enum.TryParse(Environment.GetEnvironmentVariable("REDIS_TESTS_MAINT_NOTIFICATIONS"), ignoreCase: true, out var mode) + ? mode : null; + + /// + /// Applies , if set. Call before any test-specific override. + /// + public static ConfigurationOptions ApplyMaintenanceDefault(ConfigurationOptions config) + { + if (MaintenanceNotifications is { } mode) + { + config.MaintenanceNotifications = mode; + } + return config; + } + #if NET private static int _db = 17; #else diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 4d27a206b..2276500bd 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -53,6 +53,14 @@ public InProcessTestServer(ITestOutputHelper? log = null, EndPoint? endpoint = n Tunnel = new InProcTunnel(this); } + /// + /// The highest protocol this server will agree to; lowering it lets a test have a HELLO 3 accepted + /// and answered as RESP2, which is the shape a client has to survive without being told about it. + /// + public RedisProtocol MaxProtocolVersion { get; set; } = RedisProtocol.Resp3; + + protected override RedisProtocol MaxProtocol => MaxProtocolVersion; + public Task ConnectAsync(bool withPubSub = true, bool defaultOnly = false, WriteMode writeMode = WriteMode.Default, TextWriter? log = null) => ConnectionMultiplexer.ConnectAsync(GetClientConfig(withPubSub, defaultOnly, writeMode), log); @@ -116,6 +124,7 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default Protocol = TestContext.Current.GetProtocol(), // WriteMode = (BufferedStreamWriter.WriteMode)writeMode, }; + TestConfig.ApplyMaintenanceDefault(config); if (!string.IsNullOrEmpty(Password)) config.Password = Password; config.Ssl = UseSsl; // explicitly, ignore provider defaults if (UseSsl) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs new file mode 100644 index 000000000..33ee7aeb6 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs @@ -0,0 +1,113 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Deciding what a MOVING means: where to go, or that there is nowhere to go. +/// +/// +/// Tested as a pure decision, with the resolver supplied, because the whole thing turns on DNS *changing* - +/// which no in-process fake can arrange. +/// +public class MaintenanceHandoffTests(ITestOutputHelper log) +{ + private static readonly IPAddress Retiring = IPAddress.Parse("10.129.228.140"); + private static readonly IPAddress Replacement = IPAddress.Parse("10.252.90.18"); + private static readonly DnsEndPoint Hostname = new("db.example.cloud.redislabs.com", 13486); + + private static Func> Resolves(params IPAddress[] addresses) + => (_, _) => Task.FromResult(addresses); + + [Fact] + public async Task HostnameThatHasMovedIsRecycled() + { + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.Recycle, decision.Action); + Assert.Equal(new IPEndPoint(Replacement, 13486), decision.Target); + } + + [Fact] + public async Task HostnameThatNeverMovesDoesNothing() + { + // Not a failure: the server closes the socket regardless, the reconnect re-resolves, and the relaxed + // window covers the gap. Measured on a real cluster - DNS updated three seconds *after* the close - so + // this is a normal outcome rather than a defensive branch. + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromMilliseconds(100), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: Resolves(Retiring), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + Assert.Null(decision.Target); + } + + [Fact] + public async Task AddressEndpointWithNoSuccessorHasNothingToDo() + { + // An address cannot be re-resolved, and nothing was named: there is no handoff to make. This is the + // case a cluster deployment would hit, and it is why MOVING must not simply reuse endpoint retirement - + // there would be nothing to retire *to*. + var decision = await MaintenanceHandoff.DecideAsync( + new IPEndPoint(Retiring, 13486), successor: null, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + } + + [Fact] + public async Task NamedSuccessorAsksForAReconfigure() + { + // Never observed on any real deployment - eleven routes, all explicit nulls - so this exists because the + // contract has it, not because it fires. + var successor = new IPEndPoint(Replacement, 13486); + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Retiring), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.Reconfigure, decision.Action); + Assert.Equal(successor, decision.Target); + } + + [Fact] + public async Task UnknownCurrentAddressDoesNothingRatherThanGuessing() + { + // Recycling here would mean accepting whatever DNS says *now*, which for the first several seconds is + // the address being retired - so we would hand off to the node we were told to leave. + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: null, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + } + + [Theory] + [InlineData(15, 0, 1000)] // a generous window: capped at a second, not a tenth of fifteen + [InlineData(2, 0, 200)] // a short one: a tenth, so a 2s window never spends more than 200ms + [InlineData(0, 0, 0)] // "act now" + public void JitterScalesWithTheWindowAndIsCapped(int windowSeconds, int minMilliseconds, int maxMilliseconds) + { + var random = new Random(12345); + for (int i = 0; i < 200; i++) + { + var jitter = MaintenanceHandoff.GetJitter(TimeSpan.FromSeconds(windowSeconds), random); + Assert.InRange(jitter.TotalMilliseconds, minMilliseconds, maxMilliseconds); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs new file mode 100644 index 000000000..0fad1947f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -0,0 +1,833 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Receiving the notifications: can the client see them at all, and does it make sense of the payload. Nothing +/// here asserts a *reaction* - there isn't one yet, deliberately - only that a push frame the server sends +/// arrives as an event with the right contents, and that a malformed one is dropped without collateral damage. +/// +public class MaintenanceNotificationTests(ITestOutputHelper log) +{ + private const int DefaultTimeoutMilliseconds = 5000; + + /// + /// The notifications are RESP3 push frames, so every test here forces RESP3 rather than running per + /// protocol: under RESP2 there is nothing to receive, which is covered by the opt-in tests instead. + /// + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync(ITestOutputHelper log) + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; // must be live, or the test is vacuous + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn, new EventCollector(conn)); + } + + private sealed class EventCollector + { + private readonly ConcurrentQueue _events = new(); + + public EventCollector(IConnectionMultiplexer conn) + => conn.ServerMaintenanceEvent += (_, e) => _events.Enqueue(e); + + public int Count => _events.Count; + + public IReadOnlyList All => _events.ToArray(); + + public async Task NextAsync(int timeoutMilliseconds = DefaultTimeoutMilliseconds) + { + for (int i = 0; i < timeoutMilliseconds / 25; i++) + { + if (_events.TryDequeue(out var next)) return Assert.IsType(next); + await Task.Delay(25); + } + + throw new TimeoutException("No maintenance event was received"); + } + + /// + /// Deliberately proves a negative, so it has to wait out a window in which one could have arrived. + /// + public async Task AssertNoneAsync(int milliseconds = 250) + { + await Task.Delay(milliseconds); + Assert.Empty(All); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating, MaintenanceNotificationType.Migrating)] + [InlineData(MaintenanceNotificationKind.Migrated, MaintenanceNotificationType.Migrated)] + [InlineData(MaintenanceNotificationKind.FailingOver, MaintenanceNotificationType.FailingOver)] + [InlineData(MaintenanceNotificationKind.FailedOver, MaintenanceNotificationType.FailedOver)] + public async Task ShardNotificationIsReceived(MaintenanceNotificationKind sent, MaintenanceNotificationType expected) + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var seq = server.NextMaintenanceSequenceId; + Assert.Equal(1, server.SendShardNotification(null, sent, timeSeconds: 12, shardIds: "[\"shard:1\"]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(expected, evt.NotificationType); + Assert.Equal(seq, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(12), evt.Time); + Assert.Equal("[\"shard:1\"]", evt.Payload); // opaque, carried through verbatim + Assert.Equal(server.DefaultEndPoint, evt.EndPoint); + Assert.Null(evt.NewEndPoint); + } + } + + [Fact] + public async Task MovingCarriesItsSuccessor() + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var target = new IPEndPoint(IPAddress.Loopback, 7999); + Assert.Equal(1, server.SendMoving(null, timeSeconds: 15, newEndpoint: target)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Moving, evt.NotificationType); + Assert.Equal(TimeSpan.FromSeconds(15), evt.Time); + Assert.Equal(target, evt.NewEndPoint); + + // and the deadline is projected forward, which is what a consumer actually wants + Assert.NotNull(evt.StartTimeUtc); + Assert.Equal(evt.ReceivedTimeUtc.AddSeconds(15), evt.StartTimeUtc); + } + } + + [Fact] + public async Task MovingWithNoAddressIsStillReported() + { + // the documented no-address form: a client must handle it whether or not it asked for "none", and the + // intended handling is to reconnect to what it already has - so the event must arrive, not be dropped + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendMoving(null, timeSeconds: 15, newEndpoint: null)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Moving, evt.NotificationType); + Assert.Null(evt.NewEndPoint); + Assert.Null(evt.Payload); + } + } + + [Theory] + [InlineData("?:7002")] // unknown node + [InlineData(":7002")] // node does not know its own address + [InlineData("host-1.example.com:0")] // no port to dial + public async Task MovingWithAPlaceholderYieldsNoEndpoint(string placeholder) + { + // none of these can be dialled, and none of them may be read as "the server that told us" - so the + // event arrives with the raw text and no endpoint, rather than with a plausible-looking wrong one + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MOVING", "1", "15", placeholder)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Null(evt.NewEndPoint); + Assert.Equal(placeholder, evt.Payload); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.SlotMigrating, MaintenanceNotificationType.SlotMigrating)] + [InlineData(MaintenanceNotificationKind.SlotMigrated, MaintenanceNotificationType.SlotMigrated)] + public async Task SlotNotificationIsReceived(MaintenanceNotificationKind sent, MaintenanceNotificationType expected) + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotNotification(null, sent, slots: "123,456,789-1000")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(expected, evt.NotificationType); + Assert.Equal("123,456,789-1000", evt.Payload); + Assert.Null(evt.Time); // these carry no duration + } + } + + [Fact] + public async Task AllDigitSlotListIsNotMistakenForADuration() + { + // why the type decides whether a time is expected, rather than the content: a single-slot list is + // indistinguishable from a duration by inspection + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, slots: "123")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal("123", evt.Payload); + Assert.Null(evt.Time); + } + } + + [Fact] + public async Task IntegersMayArriveAsStrings() + { + // "accept $ or : for integers" - the contract says so explicitly, and SendRawPush writes bulk strings + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MIGRATING", "42", "7", "[\"shard:2\"]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(42, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(7), evt.Time); + } + } + + [Fact] + public async Task NegativeTimeIsPreserved() + { + // a connection that joins mid-window is told what is left of it, which can be negative: that means + // "act now", so it must not be clamped or rejected + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: -3)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(TimeSpan.FromSeconds(-3), evt.Time); + Assert.Null(evt.StartTimeUtc); // nothing sensible to project + } + } + + [Fact] + public async Task LowercaseTypeNamesAreRecognized() + { + // no specification is careful about casing, so neither are we + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "failing_over", "5", "9", "[]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.FailingOver, evt.NotificationType); + } + } + + [Fact] + public async Task TrailingElementsAreTolerated() + { + // forwards compatibility is a stated requirement: a future field must not cost us the notification + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MIGRATING", "8", "20", "[\"shard:3\"]", "something-new", "and-another")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Migrating, evt.NotificationType); + Assert.Equal(8, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(20), evt.Time); + Assert.Equal("[\"shard:3\"]", evt.Payload); + } + } + + [Fact] + public async Task NestedSlotMigrationsAreRead() + { + // the shape the shipped clients read: [type, seq, [[source, target, slots], ...]]. We were dropping + // this until the prior-art cross-check, because the parser rejected non-scalar elements + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7000", "127.0.0.1:7001", "0-99"), + ("127.0.0.1:7002", "127.0.0.1:7003", "1000,2000-2500"), + ])); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrated, evt.NotificationType); + Assert.Equal(2, evt.SlotMigrations.Count); + + var first = evt.SlotMigrations[0]; + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 7000), first.Source); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 7001), first.Target); + Assert.Equal(new SlotRange(0, 99), Assert.Single(first.Slots)); + + var second = evt.SlotMigrations[1]; + Assert.Equal(2, second.Slots.Count); + Assert.Equal(new SlotRange(1000, 1000), second.Slots[0]); + Assert.Equal(new SlotRange(2000, 2500), second.Slots[1]); + Assert.Equal("1000,2000-2500", second.RawSlots); + } + } + + [Fact] + public async Task CapturedEnterpriseFramesAreUnderstood() + { + // Captured from a real Redis Cloud QA endpoint (Enterprise 8.6.2, OSS cluster API) during an actual + // slot migration on 2026-08-27. Byte for byte, except that the node addresses are rewritten into the + // private range - the lengths are preserved, so the $20 and $18 counts below are still the real ones: + // + // >3\r\n$10\r\nSMIGRATING\r\n:18\r\n$9\r\n8892-8991\r\n + // >3\r\n$9\r\nSMIGRATED\r\n:19\r\n*1\r\n*3\r\n$20\r\n10.129.228.140:13486\r\n + // $18\r\n10.252.90.18:13486\r\n$9\r\n8892-8991\r\n + // + // Things this pins, each of which was an assumption before: the type name arrives as a *bulk* string; + // the sequence number as a RESP integer rather than a string; neither cluster notification carries a + // time element; SMIGRATING's slots are a flat string; and SMIGRATED nests an array *of* triplets - one + // here - rather than a single flat triple. The fake emits this shape, so this test is the real frame. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, "8892-8991", sequenceId: 18); + + var migrating = await events.NextAsync(); + log.WriteLine(migrating.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrating, migrating.NotificationType); + Assert.Equal(18, migrating.SequenceId); + Assert.Null(migrating.Time); // no time element on the wire + Assert.Equal("8892-8991", migrating.Payload); + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [("10.129.228.140:13486", "10.252.90.18:13486", "8892-8991")], sequenceId: 19); + + var migrated = await events.NextAsync(); + log.WriteLine(migrated.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrated, migrated.NotificationType); + Assert.Equal(19, migrated.SequenceId); + Assert.Null(migrated.Time); + + var migration = Assert.Single(migrated.SlotMigrations); + Assert.Equal(new IPEndPoint(IPAddress.Parse("10.129.228.140"), 13486), migration.Source); + Assert.Equal(new IPEndPoint(IPAddress.Parse("10.252.90.18"), 13486), migration.Target); + Assert.Equal(new SlotRange(8892, 8991), Assert.Single(migration.Slots)); + } + } + + [Fact] + public async Task CapturedMovingFrameIsUnderstood() + { + // Captured from the same deployment during a maintenance_mode scenario, byte for byte: + // + // >4\r\n$6\r\nMOVING\r\n:0\r\n:15\r\n_\r\n + // + // Four elements: type, sequence *zero*, a 15-second window, and an explicit RESP3 null for the + // address - so "no replacement given, reconnect the way you connected". The proxy then closed the + // socket, which is the point of MOVING: you are told to move, and then the connection goes away. + // + // The sequence number is zero because this was the first event of that chain, not because MOVING is + // special - which is the point: zero is a legal value, so dedup tracks "have we seen one" as a separate + // bit rather than treating a stored zero as unset. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendMoving(null, timeSeconds: 15, newEndpoint: null, sequenceId: 0); + + var moving = await events.NextAsync(); + log.WriteLine(moving.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + Assert.Equal(0, moving.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(15), moving.Time); + Assert.Null(moving.NewEndPoint); // the null is the documented "use what you already have" + Assert.Null(moving.Payload); + + // and the window it opened is what covers the reconnect that follows the socket closing + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "MOVING should have relaxed timeouts"); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating, MaintenanceNotificationKind.Migrated, "27")] + [InlineData(MaintenanceNotificationKind.FailingOver, MaintenanceNotificationKind.FailedOver, "21")] + public async Task CapturedShardNotificationsAreUnderstood( + MaintenanceNotificationKind opening, + MaintenanceNotificationKind closing, + string shard) + { + // Captured from the same deployment, byte for byte, running the fault injector's maintenance_mode and + // failover scenarios: + // + // >4\r\n$9\r\nMIGRATING\r\n:0\r\n:2\r\n$6\r\n["27"]\r\n + // >3\r\n$8\r\nMIGRATED\r\n:1\r\n$6\r\n["27"]\r\n + // >4\r\n$12\r\nFAILING_OVER\r\n:0\r\n:2\r\n$6\r\n["21"]\r\n + // >3\r\n$11\r\nFAILED_OVER\r\n:1\r\n$6\r\n["21"]\r\n + // + // Both families have the same shape, and it is the shape the parser assumed: the *opening* + // notification carries a time and the *closing* one has no time element at all - which is why + // CarriesTime is keyed on the notification type rather than on sniffing whether the third element + // looks like a number. The shard list is a stringified JSON array of id strings, quotes included, and + // is carried through opaquely: nothing a client does depends on which shard it was. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var shards = $"[\"{shard}\"]"; + server.SendShardNotification(null, opening, timeSeconds: 2, shardIds: shards, sequenceId: 0); + + var open = await events.NextAsync(); + log.WriteLine(open.RawMessage ?? "(none)"); + Assert.Equal(TimeSpan.FromSeconds(2), open.Time); + Assert.Equal(shards, open.Payload); + Assert.Equal(0, open.SequenceId); + + // note 2 seconds is what the server actually said, and is shorter than the relaxed timeout floor; + // the window is clamped up to the floor rather than being honoured literally + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, $"{opening} should have relaxed timeouts"); + + server.SendShardNotification(null, closing, timeSeconds: null, shardIds: shards, sequenceId: 1); + + var closed = await events.NextAsync(); + log.WriteLine(closed.RawMessage ?? "(none)"); + Assert.Null(closed.Time); // three elements on the wire: no time to read + Assert.Equal(shards, closed.Payload); + Assert.Equal(1, closed.SequenceId); + } + } + + [Fact] + public async Task OneLogicalEventFromEveryNodeRaisesOneEvent() + { + // Observed on the real deployment: every node broadcasts a given event, and all the copies carry the + // same sequence number - the id identifies the event, not the delivery. So a two-node cluster delivers + // one migration twice. Both deliveries have to be *acted on*, because the timeout relaxation is + // per-server, but the consumer wants one callback rather than one per proxy. + var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + GetHost(server.DefaultEndPoint, out var port); + var second = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + server.Migrate((RedisKey)"elsewhere", second); // owns something, so the client connects to it + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var events = new EventCollector(conn); + using (server) + { + var muxer = (IInternalConnectionMultiplexer)conn; + Assert.True( + await Poll.UntilAsync(() => + { + var endpoints = conn.GetEndPoints(); + return endpoints.Length == 2 && endpoints.All(ep => muxer.GetServerEndPoint(ep).MaintenanceNotificationsActive); + }), + "both nodes should be connected and opted in, or there is only one delivery and this proves nothing"); + + // one send, two clients: this is the fan-out, from the server side + var delivered = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + Assert.Equal(2, delivered); + + // both servers relax - that is the internal work that has to happen per connection... + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().All(ep => muxer.GetServerEndPoint(ep).IsMaintenanceRelaxed)), + "every node that was told should have relaxed its own timeouts"); + + // ...and the consumer sees it once + Assert.Equal(1, events.Count); + var evt = await events.NextAsync(); + Assert.Equal(5, evt.SequenceId); + log.WriteLine($"{events.Count} event(s) from {delivered} deliveries: {evt.RawMessage}"); + Assert.Contains(evt.EndPoint, conn.GetEndPoints()); // whichever told us first + } + } + + /// + /// The catch-up channel: Redis Enterprise retains the most recent shard-scoped *completion* and replays it + /// to each connection that opts in, so a client that connects after a disruption still learns it happened. + /// + /// + /// Observed on RS 8.0.22 (2026-08-28), arriving coalesced into the same TCP read as the +OK for the + /// opt-in, at ~16 ms. These tests key on the *cause* of that rather than the framing: the frame is flushed + /// the instant the opt-in is processed, and our opt-in is pipelined early, so a retained frame always + /// arrives while the connection is still handshaking. + /// + public class Retention(ITestOutputHelper log) + { + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( + ITestOutputHelper log, + Action beforeConnect) + { + var server = new InProcessTestServer(log); + beforeConnect(server); // the event happens *before* anybody connects: that is the whole point + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn, new EventCollector(conn)); + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrated)] + [InlineData(MaintenanceNotificationKind.FailedOver)] + public async Task RetainedCompletionIsReplayedToANewConnection(MaintenanceNotificationKind kind) + { + var (server, conn, events) = await ConnectAsync(log, s => + s.SendShardNotification(null, kind, timeSeconds: null, shardIds: "[\"27\"]", sequenceId: 5)); + + using (server) + await using (conn) + { + // the collector is attached *after* connecting, so this asserts the frame was handled rather + // than that the event fired - which is why the relaxed window is the observable here + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "the replayed completion should have opened the post-event tail"); + Assert.Equal(MaintenanceNotificationTypeFor(kind), endpoint.ActiveMaintenanceType); + log.WriteLine($"{kind} replayed; relaxed = {endpoint.IsMaintenanceRelaxed}"); + + // ...and it must not have disturbed the handshake it arrived in the middle of, which is the + // other half of what this test is for: a push frame interleaved with our own handshake + // replies must be dispatched out-of-band rather than matched against one of them + Assert.True(conn.IsConnected); + Assert.True(await conn.GetDatabase().PingAsync() >= TimeSpan.Zero); + GC.KeepAlive(events); + } + } + + private static MaintenanceNotificationType MaintenanceNotificationTypeFor(MaintenanceNotificationKind kind) => kind switch + { + MaintenanceNotificationKind.Migrated => MaintenanceNotificationType.Migrated, + MaintenanceNotificationKind.FailedOver => MaintenanceNotificationType.FailedOver, + _ => MaintenanceNotificationType.None, + }; + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating)] + [InlineData(MaintenanceNotificationKind.FailingOver)] + [InlineData(MaintenanceNotificationKind.Moving)] + [InlineData(MaintenanceNotificationKind.SlotMigrating)] + [InlineData(MaintenanceNotificationKind.SlotMigrated)] + public async Task StartersAndSlotScopedEventsAreNotRetained(MaintenanceNotificationKind kind) + { + // The design property, asserted as a negative: the catch-up channel can only ever say "a + // disruption ended", never "one is starting". Nothing that demands action is replayed, so a + // reconnecting client cannot be told to move by a stale frame - which is what makes the MOVING + // handoff safe from replay by construction rather than by a guard. + var (server, conn, events) = await ConnectAsync(log, s => + { + if (kind == MaintenanceNotificationKind.SlotMigrated) + { + s.SendSlotMigrations(null, kind, [("127.0.0.1:7000", "127.0.0.1:7001", "50-60")], sequenceId: 5); + } + else + { + s.SendShardNotification(null, kind, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + } + }); + + using (server) + await using (conn) + { + await events.AssertNoneAsync(); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.False(endpoint.IsMaintenanceRelaxed, $"{kind} must not be retained, so nothing should have relaxed"); + } + } + + [Fact] + public async Task RetentionReplacesRatherThanAccumulates() + { + // "most recent completion", not a queue: a connection sees at most one of these however many + // events went past, which is why the client side needs no window or ring for the catch-up case + var (server, conn, events) = await ConnectAsync(log, s => + { + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"1\"]", sequenceId: 5); + s.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, null, "[\"2\"]", sequenceId: 6); + }); + + using (server) + await using (conn) + { + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed); + Assert.Equal(MaintenanceNotificationType.FailedOver, endpoint.ActiveMaintenanceType); // the later one + GC.KeepAlive(events); + } + } + + [Fact] + public async Task RetentionCanBeTurnedOff() + { + // not every deployment retains, and a test of the no-retention case should not have to reason + // about which notification kinds happen to be retained + var (server, conn, events) = await ConnectAsync(log, s => + { + s.RetainCompletions = false; + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"27\"]", sequenceId: 5); + }); + + using (server) + await using (conn) + { + await events.AssertNoneAsync(); + Assert.False(((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).IsMaintenanceRelaxed); + } + } + } + + [Theory] + [InlineData(null)] // the measured shape: the close arrives *after* the announced window + [InlineData(300)] // a less generous proxy: the close beats the window it announced + public async Task MovingClosesTheConnectionAndWeRecover(int? closeDelayMilliseconds) + { + // The defining half of MOVING, which no test covered until now: we are told to move, and then the + // socket goes away. Measured on RS (2026-08-28): the close landed at +18.4s and +16.6s against a + // declared 15s window, so the window is a floor with slack rather than a deadline - which is why the + // default case here lets the fake overshoot, and why nothing below asserts on the window expiring. + // The short case is the defensive one: a client that treats the announced window as guaranteed fails + // it. What must hold either way is that the connection comes back and the relaxed window opened by the + // notification is what covers the reconnect. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.MovingClosesConnection = true; + server.MovingCloseDelay = closeDelayMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : null; + server.SendMoving(null, timeSeconds: 1, newEndpoint: null, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "the window has to be open before the socket dies"); + + // the client must reconnect on its own; nothing here asks it to. Note the probe has to tolerate + // throwing rather than returning false: a command issued in the window between the socket dying + // and the reconnect completing raises, and that is the state being polled *out* of + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 15_000), + "the multiplexer should have re-established itself after the close"); + + log.WriteLine($"close delay {closeDelayMilliseconds?.ToString() ?? "announced+slack"}: reconnected, relaxed = {endpoint.IsMaintenanceRelaxed}"); + } + } + + [Fact] + public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() + { + // The point of D6: today a MOVING is survivable because the socket eventually closes and we reconnect, + // but the announced window goes unused. Here nothing closes the connection - MovingClosesConnection is + // deliberately left off - so the *only* thing that can replace it is our own handoff. + // + // The named-successor branch is what is exercised, because the in-process transport has no socket and so + // no current address for the DNS branch to compare against; the deciding half of that branch is covered + // by MaintenanceHandoffTests against a supplied resolver. What this proves is the acting half: drain, + // drop, and come back. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var optInsBefore = server.TotalMaintenanceOptIns; + Assert.Equal(1, optInsBefore); // the handshake's own opt-in, so the increment below means something + + // A handoff has to be *visible*: the replacement raises ConnectionRestored, so reporting nothing for + // the drop would leave a consumer tracking connection state with an unpaired restore. + var failures = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (failures) failures.Add(e.FailureType); + }; + + server.SendMoving(null, timeSeconds: 2, newEndpoint: server.DefaultEndPoint, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + Assert.Equal(server.DefaultEndPoint, moving.NewEndPoint); + + // a fresh connection re-sends the opt-in, which is how a recycle is visible from the server's side + Assert.True( + await Poll.UntilAsync(() => server.TotalMaintenanceOptIns > optInsBefore, timeoutMilliseconds: 15_000), + "the handoff should have replaced the connection without the server closing it"); + + log.WriteLine($"opt-ins: {optInsBefore} -> {server.TotalMaintenanceOptIns}"); + + Assert.True( + await Poll.UntilAsync( + () => + { + lock (failures) return failures.Contains(ConnectionFailureType.MaintenanceHandoff); + }, + timeoutMilliseconds: 5_000), + "the recycle should be reported as a MaintenanceHandoff rather than silently"); + + lock (failures) + { + log.WriteLine($"reported: {string.Join(", ", failures)}"); + + // and *only* as that: reporting it as a socket failure would put planned maintenance into + // everybody's fault dashboards + Assert.DoesNotContain(ConnectionFailureType.SocketFailure, failures); + Assert.DoesNotContain(ConnectionFailureType.SocketClosed, failures); + } + + // ...and the replacement is usable, which is the only outcome a caller cares about + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 15_000), + "the client should be serving commands on the replacement connection"); + } + } + + [Fact] + public async Task MalformedTripletIsSkippedNotFatal() + { + // one bad entry must not lose the migrations we could have applied - the same choice go-redis makes + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7000", "127.0.0.1:7001", "not-a-slot-list"), // kept, but with no parsed slots + ("127.0.0.1:7002", "?", "50-60"), // an unnameable target, kept with a null Target + ("127.0.0.1:7004", "127.0.0.1:7005", "70"), + ]); + + var evt = await events.NextAsync(); + Assert.Equal(3, evt.SlotMigrations.Count); + Assert.Empty(evt.SlotMigrations[0].Slots); + Assert.Equal("not-a-slot-list", evt.SlotMigrations[0].RawSlots); // raw form survives + Assert.Null(evt.SlotMigrations[1].Target); + Assert.Equal(new SlotRange(50, 60), Assert.Single(evt.SlotMigrations[1].Slots)); + Assert.Equal(new SlotRange(70, 70), Assert.Single(evt.SlotMigrations[2].Slots)); + } + } + + [Fact] + public async Task MissingSequenceIdStillReportsTheNotification() + { + // stricter-than-shipped-clients was the bug here: go-redis length-checks these at two elements and + // reads no sequence number at all, so dropping a frame for want of one loses a real disruption + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + // two elements, with an unreadable seq - the floor go-redis works to + Assert.True(server.SendRawPush(null, "MIGRATING", "not-a-number") > 0); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Migrating, evt.NotificationType); + } + } + + [Theory] + [InlineData("NOT_A_REAL_KIND", "1", "5")] // a type we don't know + [InlineData("NOT_A_REAL_KIND")] // ...and one with nothing else at all + public async Task MalformedNotificationIsDroppedAndTheConnectionSurvives(params string[] parts) + { + // the important half of this feature's safety: an unparseable push frame must be consumed and + // forgotten. If it fell through to the command matcher it would steal the next command's reply, and + // everything after that would be answering the wrong question + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.True(server.SendRawPush(null, parts) > 0); + await events.AssertNoneAsync(); + + // the connection is not merely alive but *in sync*: each reply is the answer to its own command + var db = conn.GetDatabase(); + for (int i = 0; i < 10; i++) + { + await db.StringSetAsync($"maint-sync-{i}", i); + } + + for (int i = 0; i < 10; i++) + { + Assert.Equal(i, (int)await db.StringGetAsync($"maint-sync-{i}")); + } + } + } + + [Fact] + public async Task NotificationsArriveInterleavedWithCommands() + { + // a push can land at any point, including between a command and its reply; nothing may be lost or + // mismatched either way + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var db = conn.GetDatabase(); + var pending = new List>(); + for (int i = 0; i < 50; i++) + { + await db.StringSetAsync($"maint-interleave-{i}", i); + pending.Add(db.StringGetAsync($"maint-interleave-{i}")); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: i); + } + + var values = await Task.WhenAll(pending); + Assert.Equal(Enumerable.Range(0, 50), values.Select(x => (int)x)); + + // and every notification arrived + for (int i = 0; i < 50; i++) + { + await events.NextAsync(); + } + + log.WriteLine($"{values.Length} commands and 50 notifications, all accounted for"); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs new file mode 100644 index 000000000..7da72d32d --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -0,0 +1,251 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// The client half of the opt-in: what we send during handshake, and what we make of the answer. The server +/// half is . +/// +[RunPerProtocol] +public class MaintenanceOptInClientTests(ITestOutputHelper log) +{ + private static InProcessTestServer CreateServer(ITestOutputHelper log) => new(log); + + private static ConfigurationOptions Config(InProcessTestServer server, MaintenanceNotificationMode mode) + { + var config = server.GetClientConfig(defaultOnly: true); + config.MaintenanceNotifications = mode; + return config; + } + + private static bool IsActive(IConnectionMultiplexer conn, InProcessTestServer server) + => ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).MaintenanceNotificationsActive; + + private static List OptedIn(InProcessTestServer server) + { + var found = new List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } + + /// + /// Asserts that Enabled refused: either the connect threw, or it returned a connection that does + /// not stay usable. + /// + /// + /// Both outcomes are the same refusal, and which one a caller sees is a race: the reconcile records the + /// failure from the handshake-completion path, which can land either side of ConnectAsync deciding + /// it has a connection. Asserting only the throw made this flaky on a two-core runner, and asserting the + /// throw is not the point - not being left with a working connection is. + /// + private static async Task AssertRefusedAsync(ConfigurationOptions config, ITestOutputHelper log) + { + ConnectionMultiplexer? conn = null; + try + { + conn = await ConnectionMultiplexer.ConnectAsync(config); + } + catch (RedisConnectionException ex) + { + log.WriteLine($"refused at connect: {ex.Message}"); + return; + } + + await using (conn) + { + var endpoint = conn.GetEndPoints().Single(); + var unusable = await Poll.UntilAsync(() => !conn.GetServer(endpoint).IsConnected); + log.WriteLine($"connected, then unusable: {unusable}"); + Assert.True(unusable, "the connection should not have stayed usable"); + } + } + + [Fact] + public async Task HandshakeOptsInWhenAuto() + { + using var server = CreateServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Auto)); + + var resp3 = TestContext.Current.IsResp3(); + log.WriteLine($"protocol: {TestContext.Current.GetProtocol()}, opted in: {OptedIn(server).Count}"); + + // RESP3-only by design: under RESP2 the server could accept the request and then never deliver + // anything, so we don't ask - and under Auto that is simply the feature being off + Assert.Equal(resp3, IsActive(conn, server)); + Assert.Equal(resp3 ? 1 : 0, OptedIn(server).Count); + + if (resp3) + { + // a bare ON, so the server chooses; nothing here invents an endpoint type + Assert.Null(Assert.Single(OptedIn(server)).MovingEndpointType); + } + + // ...and the connection is entirely usable either way + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task DisabledSendsNothing() + { + using var server = CreateServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Disabled)); + + Assert.Empty(OptedIn(server)); + Assert.False(IsActive(conn, server)); + } + + [Fact] + public async Task DefaultIsOff() + { + // the library default has to be Disabled: OSS Redis, Valkey and Garnet don't know the subcommand, and + // an unsolicited error reply on every connection would be a poor first impression + using var server = CreateServer(log); + var config = server.GetClientConfig(defaultOnly: true); // untouched, so whatever the provider says + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Equal(MaintenanceNotificationMode.Disabled, config.MaintenanceNotifications); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.UnknownSubcommand)] + [InlineData(MaintenanceNotificationSupport.Disabled)] + public async Task AutoToleratesAServerThatRefuses(MaintenanceNotificationSupport support) + { + // the whole point of Auto: ask, and carry on regardless. This is what lets the entire test suite run + // with the opt-in on against servers that will never send a notification + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Auto)); + + Assert.False(IsActive(conn, server)); + Assert.Empty(OptedIn(server)); + Assert.True(conn.GetServer(server.DefaultEndPoint).IsConnected); + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task EnabledFailsAgainstAServerThatRefuses() + { + // Enabled is a requirement, not a preference: a connection that silently won't deliver notifications + // is not the connection that was asked for + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.MaintenanceNotifications = MaintenanceNotificationSupport.UnknownSubcommand; + + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task AutoIsOffWhenTheServerDowngradesToResp2() + { + // the case that makes this RESP3-only: the server accepts the opt-in and then answers HELLO as RESP2, + // so nothing would ever arrive. We asked, the server said OK, and we still treat it as off + using var server = CreateServer(log); + server.MaxProtocolVersion = RedisProtocol.Resp2; + var config = Config(server, MaintenanceNotificationMode.Auto); + config.Protocol = RedisProtocol.Resp3; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Single(OptedIn(server)); // the server took it... + Assert.False(IsActive(conn, server)); // ...and we know better than to believe it + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task EnabledFailsWhenTheServerDowngradesToResp2() + { + using var server = CreateServer(log); + server.MaxProtocolVersion = RedisProtocol.Resp2; + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.Protocol = RedisProtocol.Resp3; + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task EnabledFailsWhenResp2WasOurOwnChoice() + { + // no exemption for a contradiction we configured ourselves: requiring a RESP3-only feature over a + // RESP2 connection cannot be honoured, and half-honouring it silently is what Enabled exists to avoid + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.Protocol = RedisProtocol.Resp2; + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task AutoIsHappyOnResp2() + { + // the counterpart: Auto is best-effort, so an explicit RESP2 just means the feature is off + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.Protocol = RedisProtocol.Resp2; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Empty(OptedIn(server)); + Assert.False(IsActive(conn, server)); + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task OptInIsReArmedOnReconnect() + { + // per-connection state, so a reconnect that didn't re-send it would leave us silently unsubscribed + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.AllowSimulateConnectionFailure = true; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(IsActive(conn, server)); + + var before = server.TotalMaintenanceOptIns; + conn.GetServer(server.DefaultEndPoint).SimulateConnectionFailure(SimulatedFailureType.All); + await UntilCondition(() => server.TotalMaintenanceOptIns > before); + + log.WriteLine($"opt-ins: {before} -> {server.TotalMaintenanceOptIns}"); + Assert.True(server.TotalMaintenanceOptIns > before, "the opt-in should be sent again on the new connection"); + + // ...and then wait for *our* side of it. The count above is the server having processed the opt-in, + // whereas IsActive is us having processed its reply - a beat later, so asserting it directly is a race + // that only shows up when the runner is starved of cores. + await UntilCondition(() => IsActive(conn, server)); + Assert.True(IsActive(conn, server), "the feature should be live again on the replacement connection"); + } + + private static async Task UntilCondition(System.Func condition, int timeoutMilliseconds = 5000) + { + for (int i = 0; i < timeoutMilliseconds / 50 && !condition(); i++) + { + await Task.Delay(50); + } + } + + private static async Task Set(IConnectionMultiplexer conn) + { + var db = conn.GetDatabase(); + await db.StringSetAsync("maint-optin", "value"); + return await db.StringGetAsync("maint-optin"); + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs new file mode 100644 index 000000000..c44296df0 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs @@ -0,0 +1,322 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Timeout relaxation: a server that announces a disruption gets its command timeouts raised for the duration, +/// as a floor and never a reduction. The window arithmetic is asserted through internals rather than by waiting +/// out real seconds - the interesting properties are "does it open, extend, close, and expire correctly", and a +/// test that sleeps for 30s to watch a cap expire is a test nobody runs. +/// +public class MaintenanceRelaxationTests(ITestOutputHelper log) +{ + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + Action? configure = null) + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + configure?.Invoke(config); + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + private static ServerEndPoint Endpoint(IConnectionMultiplexer conn, InProcessTestServer server) + => ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + + /// + /// The notification arrives on the connection's read loop, so a test that sends one has to let it land. + /// + private static async Task UntilRelaxedAsync(ServerEndPoint endpoint, bool expected, int timeoutMilliseconds = 5000) + { + for (int i = 0; i < timeoutMilliseconds / 25 && endpoint.IsMaintenanceRelaxed != expected; i++) + { + await Task.Delay(25); + } + return endpoint.IsMaintenanceRelaxed; + } + + [Fact] + public async Task NoWindowMeansNoChange() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + Assert.False(endpoint.IsMaintenanceRelaxed); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + } + } + + [Fact] + public async Task OpeningNotificationRelaxesTimeouts() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + + Assert.True(await UntilRelaxedAsync(endpoint, true), "should be relaxed"); + + // 10s relaxed against a 5s configured timeout + Assert.Equal(10_000, endpoint.GetEffectiveTimeoutMilliseconds(5_000)); + } + } + + [Fact] + public async Task RelaxationIsAFloorAndNeverAReduction() + { + // a caller with a generous timeout keeps it; this is explicit in the contract + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.Equal(60_000, endpoint.GetEffectiveTimeoutMilliseconds(60_000)); + } + } + + [Fact] + public async Task ClosingNotificationLeavesThePostEventTail() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + await Task.Delay(250); // let it land; the tail means "still relaxed", so there is no flag to await + + // the server said it finished, and we are *still* relaxed - because that is when every other + // client that got the same notification comes back + Assert.True(endpoint.IsMaintenanceRelaxed, "the post-event tail should still be running"); + Assert.Equal(10_000, endpoint.GetEffectiveTimeoutMilliseconds(5_000)); + } + } + + [Fact] + public async Task ClosingNotificationEndsItWhenThereIsNoTail() + { + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false), "should have ended"); + } + } + + [Fact] + public async Task WindowExpiresOnItsOwn() + { + // the cap and the ordinary deadline share a mechanism; a one-second floor lets us watch it expire + // without a test that sleeps for half a minute + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero; + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + + // asks for 20s, capped to 1s + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.False(await UntilRelaxedAsync(endpoint, false), "the cap should have ended it"); + } + } + + [Fact] + public async Task NoTailAfterACapExpiry() + { + // if the window ended on the cap we never learned the event finished, so extending past the backstop + // would defeat the backstop + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.FromSeconds(30); // would be very visible + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.False(await UntilRelaxedAsync(endpoint, false), "no tail should follow a capped window"); + } + } + + [Fact] + public async Task ShorterNotificationDoesNotCutAnOpenWindowShort() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + + // a long window, then a short one on top: the short one must not shorten it + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 25); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 1); + await Task.Delay(1500); + + Assert.True(endpoint.IsMaintenanceRelaxed, "the longer window should still be in force"); + } + } + + [Fact] + public async Task ReplayedSequenceIdIsIgnored() + { + // our own invention, so kept conservative: an id we have already acted on cannot extend a window + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero; + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + var seq = server.NextMaintenanceSequenceId; + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + Assert.False(await UntilRelaxedAsync(endpoint, false)); + + // replaying the same id must not reopen anything + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20, sequenceId: seq); + await Task.Delay(250); + Assert.False(endpoint.IsMaintenanceRelaxed, "a replayed id should not reopen the window"); + + // ...but a genuinely new one still does + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true), "a new id should still work"); + } + } + + [Fact] + public async Task RelaxationDoesNotSuppressConnectionFailure() + { + // the regression that would matter most: relaxation must touch command timeouts only. If it reached + // keep-alive or failure detection, a server that died mid-maintenance would linger for the window - + // turning a latency mitigation into an availability regression + var (server, conn) = await ConnectAsync(log, config => + { + config.AllowSimulateConnectionFailure = true; + config.MaintenanceRelaxedTimeout = TimeSpan.FromMinutes(5); // absurdly generous, deliberately + config.MaintenanceRelaxedWindowMax = TimeSpan.FromMinutes(5); + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 300); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + var failures = 0; + conn.ConnectionFailed += (_, _) => Interlocked.Increment(ref failures); + conn.GetServer(server.DefaultEndPoint).SimulateConnectionFailure(SimulatedFailureType.All); + + for (int i = 0; i < 100 && Volatile.Read(ref failures) == 0; i++) + { + await Task.Delay(25); + } + + log.WriteLine($"connection failures observed: {Volatile.Read(ref failures)}"); + Assert.True(Volatile.Read(ref failures) > 0, "a dead connection must still be noticed during a relaxed window"); + } + } + + [Theory] + [InlineData(true, MaintenanceNotificationType.Migrating)] + [InlineData(false, MaintenanceNotificationType.None)] + public async Task TimeoutReportsWhetherMaintenanceWasInForce(bool announce, MaintenanceNotificationType expected) + { + // "timeout" and "timeout during an announced migration" call for very different reactions from + // whoever reads the log, so the fault says which it was + var (server, conn) = await ConnectAsync(log, config => + { + config.AsyncTimeout = 200; + config.SyncTimeout = 200; + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(30); + config.HeartbeatInterval = TimeSpan.FromMilliseconds(100); + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + if (announce) + { + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + } + + server.SetLatency(TimeSpan.FromSeconds(5)); // longer than any timeout in play + var ex = await Assert.ThrowsAsync(() => conn.GetDatabase().StringGetAsync("maint-fault")); + log.WriteLine($"{ex.MaintenanceType}: {ex.Message}"); + Assert.Equal(expected, ex.MaintenanceType); + + // and it reaches the circuit-breaker/retry surface the same way + var context = new Availability.FaultContext(ex); + Assert.Equal(expected, context.MaintenanceType); + + server.SetLatency(TimeSpan.Zero); + } + } + + [Fact] + public async Task CommandsStillWorkThroughAWindow() + { + // relaxation is invisible to anything that is not timing out + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + var db = conn.GetDatabase(); + for (int i = 0; i < 20; i++) + { + await db.StringSetAsync($"relax-{i}", i); + } + for (int i = 0; i < 20; i++) + { + Assert.Equal(i, (int)await db.StringGetAsync($"relax-{i}")); + } + + // including the synchronous path, which has its own re-wait loop + Assert.Equal(7, (int)db.StringGet("relax-7")); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs new file mode 100644 index 000000000..5cddbb1eb --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs @@ -0,0 +1,400 @@ +using System; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Server; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Non-parallel deliberately. Several of these wait out a jittered refresh, and the retirement one depends on +/// the doomed server being *idle* - and IsIdle counts outstanding work, which a heartbeat ping in flight +/// supplies. On a loaded machine those complete slowly enough that pruning is starved, which is a real property +/// of the policy rather than a flaw in the test: see the design notes on why a usage-based grace rule was +/// dropped for exactly this reason. +/// +/// Reacting to a completed slot migration by re-reading the topology, rather than waiting to be told by a +/// -MOVED. Asserted from the server's side - did another CLUSTER command actually arrive - because +/// that is the thing the fleet pays for, and the thing a scoping bug would multiply. +/// +/// +[Collection(NonParallelCollection.Name)] +public class MaintenanceTopologyRefreshTests(ITestOutputHelper log) +{ + /// + /// Counts inbound CLUSTER commands, so a test can tell a refresh happened from the outside. + /// + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands, _subscribeCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public int SubscribeCommands => Volatile.Read(ref _subscribeCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0) + { + var command = request.GetString(0); + if (string.Equals(command, "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + else if (string.Equals(command, "ssubscribe", StringComparison.OrdinalIgnoreCase) + || string.Equals(command, "subscribe", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _subscribeCommands); + } + } + + return base.Execute(client, in request); + } + } + + private static async Task<(CountingServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + Action? configure = null, + MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + configure?.Invoke(server); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = mode; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + /// + /// The jitter is up to a second, so a refresh is not immediate by design. + /// + private static Task UntilRefreshedAsync(CountingServer server, int before) + => Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 5000); + + [Fact] + public async Task SlotsMovingAwayFromUsTriggersARefresh() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", "0-99"), + ]); + + Assert.True(await UntilRefreshedAsync(server, before), "a refresh should follow slots leaving us"); + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + } + } + + [Fact] + public async Task SlotsMovingBetweenOtherNodesDoesNotTriggerARefresh() + { + // the whole herd argument: every node reports the same movements, so most notifications are about + // somebody else. Refreshing on those means every client in the fleet re-reads topology whenever any + // shard moves anywhere + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7100", "127.0.0.1:7101", "0-99"), + ("127.0.0.1:7102", "127.0.0.1:7103", "100-199"), + ]); + + await Task.Delay(2000); // longer than the jitter, so "not yet" cannot be mistaken for "never" + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task SourceIsMatchedByAnnouncedIdentityNotJustAddress() + { + // A node answers to its address *and* its announced hostname, and the delta may name either, so the + // scoping test resolves through the identity map rather than comparing endpoints as text. Announced + // before connecting, because the handshake's own topology read is what registers the identity - a + // hostname set afterwards is not known to us until something re-reads it. + const string Hostname = "node-1.redis.example.com"; + var (server, conn) = await ConnectAsync(log, s => s.SetHostname(s.DefaultEndPoint, Hostname)); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"{Hostname}:{port}", $"127.0.0.1:{port + 1}", "0-99"), + ]); + + Assert.True(await UntilRefreshedAsync(server, before), "the hostname form should resolve to us"); + } + } + + [Fact] + public async Task ProxyStyleCompletionDoesNotTriggerARefresh() + { + // MIGRATED/FAILED_OVER arrive in proxied deployments addressed as a single endpoint, where there is no + // topology for a refresh to learn - so they get relaxation and nothing else + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + + await Task.Delay(2000); + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task ShardedSubscriptionOnAMovedSlotIsReEstablished() + { + // mostly belt-and-braces: the server also sends an unsolicited SUNSUBSCRIBE for this, which we already + // act on. What this adds is being pre-emptive when SMIGRATED lands first, and covering the case where + // the unsubscribe never arrives - where the only other symptom is messages silently stopping + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("resub-channel"); + await sub.SubscribeAsync(channel, (_, _) => { }); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + log.WriteLine($"channel hashes to slot {slot}"); + + var before = server.SubscribeCommands; + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{slot}"), + ]); + + Assert.True( + await Poll.UntilAsync(() => server.SubscribeCommands > before), + "the sharded subscription should have been re-established"); + } + } + + [Fact] + public async Task ShardedSubscriptionOnAnUnaffectedSlotIsLeftAlone() + { + // knowing *which* slots moved is the advantage over the unsolicited-unsubscribe path: only the + // affected channels are touched, rather than everything subscribed on this server + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("untouched-channel"); + await sub.SubscribeAsync(channel, (_, _) => { }); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + var otherSlot = slot == 0 ? 1 : 0; + var before = server.SubscribeCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{otherSlot}"), + ]); + + await Task.Delay(2000); + log.WriteLine($"subscribe commands: {before} -> {server.SubscribeCommands} (slot {slot} vs moved {otherSlot})"); + Assert.Equal(before, server.SubscribeCommands); + } + } + + [Fact] + public async Task OrdinaryPubSubIsNotSlotBoundAndIsLeftAlone() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + await sub.SubscribeAsync(RedisChannel.Literal("plain-channel"), (_, _) => { }); + + var before = server.SubscribeCommands; + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", "0-16383"), // everything moves + ]); + + await Task.Delay(2000); + log.WriteLine($"subscribe commands: {before} -> {server.SubscribeCommands}"); + Assert.Equal(before, server.SubscribeCommands); + } + } + + [Theory] + [InlineData(MaintenanceNotificationMode.Disabled)] // only the unsolicited SUNSUBSCRIBE path can act + [InlineData(MaintenanceNotificationMode.Enabled)] // both paths see the same migration + public async Task RealMigrationRecoversTheSubscriptionWithoutStorming(MaintenanceNotificationMode mode) + { + // Until the fake announced its own migrations this could not be tested at all, and it is the case that + // matters: a real slot migration produces *both* an unsolicited SUNSUBSCRIBE (ordinary cluster + // behaviour, sent to every subscriber) and SMIGRATED (only to clients that opted in). Both paths lead + // to ResubscribeToServer, so the question is whether the subscription ends up established exactly + // once rather than twice or not at all. + var (server, conn) = await ConnectAsync(log, mode: mode); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var doomed = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("migrating-channel"); + var received = 0; + await sub.SubscribeAsync(channel, (_, _) => Interlocked.Increment(ref received)); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + var before = server.SubscribeCommands; + + server.NotifyOnMigrate = true; + server.Migrate(slot, doomed); // the whole realistic sequence, from one call + + Assert.True(await Poll.UntilAsync(() => server.SubscribeCommands > before), "should have resubscribed"); + await Task.Delay(2000); // and settle, so a second resubscribe would show up + + var resubscribes = server.SubscribeCommands - before; + log.WriteLine($"{mode}: {resubscribes} (re)subscribe command(s) after a real migration"); + + // The property is boundedness, not an absolute count: the unsolicited-unsubscribe path has its own + // pre-existing retry-and-redirect behaviour. Measured at 4 with notifications off and 5 with them + // on - the extra one being the fallback acting on a subscription that path left attached to + // nothing, which is the feature working. What must not happen is one attempt per notification. + // Bounded, not exact: the unsolicited-unsubscribe path retries and follows redirects, and how + // many of those land depends on timing (measured 4 unloaded, 7 on a constrained runner). The + // property is that the count does not scale with notifications - one migration, a handful of + // attempts - so the bound is deliberately loose rather than pinned to a number that will drift. + Assert.InRange(resubscribes, 1, 15); + + // Delivery is asserted by publishing *repeatedly*, because pub/sub is fire and forget: a message + // published while the subscription is still in flux is simply dropped. Losing messages during the + // tremor is expected; never delivering again is not, and one publish cannot tell those apart. + var delivered = await Poll.UntilAsync( + () => + { + if (Volatile.Read(ref received) > 0) return true; + conn.GetSubscriber().Publish(channel, "hello"); + return Volatile.Read(ref received) > 0; + }, + timeoutMilliseconds: 10_000, + pollMilliseconds: 250); + + log.WriteLine($"{mode}: delivered after migration = {delivered}"); + Assert.True(delivered, "the subscription should deliver again once things settle"); + } + } + + [Fact] + public async Task NodeThatLeavesTheClusterIsRetired() + { + // This is also the regression test for idleness counting *caller* work only. Before that, the + // reconfigure's own probes to the departed node piled into its backlog (~170 per pass, unbounded), so + // it looked busy because we were looking for it, and could never be retired. + + // The narrowed form of D5's "retire endpoints serving no slots". Serving nothing is *not* the + // condition: a node still listed in CLUSTER NODES is a live member that may be given slots again, and + // dropping its connection would be churn (go-redis does not). Having *left* the cluster is the + // condition, and the notification-driven refresh is what makes us notice. + EndPoint doomed = null!; + var (server, conn) = await ConnectAsync(log, configure: s => + { + GetHost(s.DefaultEndPoint, out var p); + doomed = s.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, p + 1)); + s.Migrate((RedisKey)"leaving", doomed); // owns something, so it is discovered at connect + }); + + using (server) + await using (conn) + { + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + GetHost(server.DefaultEndPoint, out var basePort); + var third = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, basePort + 2)); + + // hand its slot back and then remove it from the cluster outright + server.NotifyOnMigrate = true; + server.Migrate((RedisKey)"leaving", server.DefaultEndPoint); + Assert.True(server.RemoveNode(doomed), "the node should have been removed from the fake"); + + // Pruning wants several consecutive generations of absence. Driving those by *awaiting* topology + // passes rather than by sending notifications and sleeping: the notification path is covered by + // the tests above, and depending on it here would make this test wait out a jittered refresh per + // generation - which is what made it fail under a two-core runner. What is being tested is the + // retirement, so drive the generations deterministically. + // Retirement also requires the server to be *idle*, so a busy moment can defer it past a given + // pass - which is why this drives passes until it happens rather than asserting after a fixed + // count. Bounded, so a regression fails rather than hangs. + GC.KeepAlive(third); + var mux = (ConnectionMultiplexer)conn; + for (int i = 0; i < 20 && conn.GetEndPoints().Contains(doomed); i++) + { + await mux.ReconfigureAsync(first: false, reconfigureAll: true, log: null, blame: null, cause: $"test-generation-{i}"); + await Task.Delay(50); // retirement drains before removing, so give it a moment to complete + + } + + log.WriteLine($"endpoints: {string.Join(", ", conn.GetEndPoints().Select(x => x.ToString()))}"); + Assert.DoesNotContain(doomed, conn.GetEndPoints()); + } + } + + [Fact] + public async Task RefreshIsCoalescedRatherThanRepeated() + { + // A burst of notifications must not be a burst of topology reads. Note ReconfigureIfNeeded's own + // coalescing is *not* what achieves this: it declines only while a refresh is actually in flight, and + // the jitter spreads a burst out far enough that each pass finishes before the next starts. This test + // is what caught that, and why the coalescing happens before the delay rather than after. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + for (int i = 0; i < 10; i++) + { + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{i * 10}-{(i * 10) + 9}"), + ]); + } + + Assert.True(await UntilRefreshedAsync(server, before)); + await Task.Delay(2000); // let any stragglers land + + // a refresh reads both SLOTS and NODES, so allow a small multiple - the point is that ten + // notifications did not produce ten topology passes + var added = server.ClusterCommands - before; + log.WriteLine($"{added} cluster command(s) for 10 notifications"); + Assert.InRange(added, 1, 8); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs index 748589e2b..83ded0d7b 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Threading.Tasks; @@ -107,6 +107,6 @@ public void LiteralsResolveToExpectedStorageKinds() // only two storage flavors should ever appear; a third means something new crept in (e.g. a numeric // literal => Int64), which we want to notice Assert.Equal(2, byKind.Count); - Assert.Equal(22, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes + Assert.Equal(23, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes } } diff --git a/tests/StackExchange.Redis.Tests/TestBase.cs b/tests/StackExchange.Redis.Tests/TestBase.cs index c59039478..505f2bb4e 100644 --- a/tests/StackExchange.Redis.Tests/TestBase.cs +++ b/tests/StackExchange.Redis.Tests/TestBase.cs @@ -454,7 +454,7 @@ public static ConnectionMultiplexer CreateDefault( log ??= localLog = new StringWriter(); try { - var config = ConfigurationOptions.Parse(configuration); + var config = TestConfig.ApplyMaintenanceDefault(ConfigurationOptions.Parse(configuration)); if (disabledCommands != null && disabledCommands.Length != 0) { config.CommandMap = CommandMap.Create([.. disabledCommands], false); diff --git a/toys/MaintenanceWatch/MaintenanceWatch.csproj b/toys/MaintenanceWatch/MaintenanceWatch.csproj new file mode 100644 index 000000000..b400a6ced --- /dev/null +++ b/toys/MaintenanceWatch/MaintenanceWatch.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + + $(NoWarn);SER010 + + + + + + + diff --git a/toys/MaintenanceWatch/Program.cs b/toys/MaintenanceWatch/Program.cs new file mode 100644 index 000000000..f75aacdec --- /dev/null +++ b/toys/MaintenanceWatch/Program.cs @@ -0,0 +1,115 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; +using StackExchange.Redis.Maintenance; + +// Watches a real deployment for server-native maintenance notifications and prints what we made of them. +// +// The point of this tool is *evidence*: that we asked for notifications, that they arrive, and that each one +// was understood - so every event prints both our parsed view and the raw payload it came from, and any +// disagreement between the two is visible rather than inferred. +// +// dotnet run --project toys/MaintenanceWatch -- "my-endpoint:6379,user=...,password=..." +// +// Notes: +// - RESP3 is required; this forces it rather than relying on the endpoint's provider defaults. +// - The opt-in mode is forced to Auto, so an endpoint that does not support notifications still connects. +// - Nothing here reacts to a notification; observing is the whole job. +if (args.Length == 0) +{ + Console.Error.WriteLine("usage: MaintenanceWatch [--enabled]"); + return 1; +} + +var config = ConfigurationOptions.Parse(args[0]); +config.Protocol = RedisProtocol.Resp3; +config.MaintenanceNotifications = args.Contains("--enabled") + ? MaintenanceNotificationMode.Enabled // refuse to run if the server will not deliver + : MaintenanceNotificationMode.Auto; +config.AbortOnConnectFail = false; + +// note ToString() includes the password by default; the library's own logging masks it (see +// LoggerExtensions), but this line is ours, so it has to ask +Console.WriteLine($"connecting: {config.ToString(includePassword: false)}"); +Console.WriteLine($"defaults provider: {config.Defaults}"); +Console.WriteLine($"maintNotifications={config.MaintenanceNotifications}, relaxed={config.MaintenanceRelaxedTimeout.TotalSeconds}s," + + $" windowMax={config.MaintenanceRelaxedWindowMax.TotalSeconds}s, postEvent={config.MaintenancePostEventRelaxedDuration.TotalSeconds}s"); + +// the connect log carries the negotiation half: whether the opt-in was sent, and how the server answered +await using var conn = await ConnectionMultiplexer.ConnectAsync(config, Console.Out); + +var count = 0; +conn.ServerMaintenanceEvent += (_, e) => +{ + var n = Interlocked.Increment(ref count); + Console.WriteLine(); + Console.WriteLine($"=== maintenance event #{n} at {DateTime.UtcNow:HH:mm:ss.fff}Z ==="); + Console.WriteLine($" raw: {e.RawMessage}"); + + if (e is not PushMaintenanceEvent push) + { + // e.g. the Azure pub/sub channel; not what this tool is for, but worth seeing rather than hiding + Console.WriteLine($" (not a push notification: {e.GetType().Name})"); + return; + } + + Console.WriteLine($" type: {push.NotificationType} seq: {push.SequenceId} from: {push.EndPoint}"); + Console.WriteLine($" time: {(push.Time is { } t ? $"{t.TotalSeconds}s" : "(none)")}" + + $" startsAt: {(push.StartTimeUtc is { } at ? $"{at:HH:mm:ss}Z" : "(n/a)")}"); + + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + // null is a documented outcome, not a parse failure: reconnect to what you already have + Console.WriteLine($" moving to: {push.NewEndPoint?.ToString() ?? "(no address given)"}"); + } + + if (push.Payload is { } payload) + { + Console.WriteLine($" payload: {payload}"); + } + + foreach (var migration in push.SlotMigrations) + { + var slots = migration.Slots.Count == 0 + ? $"(unparsed: '{migration.RawSlots}')" + : string.Join(",", migration.Slots.Select(x => x.From == x.To ? $"{x.From}" : $"{x.From}-{x.To}")); + Console.WriteLine($" slots {slots}: {migration.Source?.ToString() ?? "?"} -> {migration.Target?.ToString() ?? "?"}"); + } +}; + +// a fault during an announced disruption says so, which is the other half of "we understood it" +conn.ErrorMessage += (_, e) => Console.WriteLine($"[error] {e.EndPoint}: {e.Message}"); +conn.ConnectionFailed += (_, e) => Console.WriteLine($"[failed] {e.EndPoint}: {e.FailureType} {e.Exception?.Message}"); +conn.ConnectionRestored += (_, e) => Console.WriteLine($"[restored] {e.EndPoint}"); + +Console.WriteLine(); +Console.WriteLine("watching; a little traffic keeps the connection interesting. Ctrl+C to stop."); + +var db = conn.GetDatabase(); +var key = $"maintenance-watch:{Guid.NewGuid():N}"; +while (true) +{ + try + { + await db.StringSetAsync(key, (RedisValue)DateTime.UtcNow.Ticks); + _ = await db.StringGetAsync(key); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // note RedisTimeoutException derives from TimeoutException, *not* RedisException - so a + // `catch (RedisException)` would silently miss exactly the case this tool exists to show + // MaintenanceType is the payoff here: "timeout" versus "timeout during an announced failover" + var maintenance = ex switch + { + RedisTimeoutException timeout => timeout.MaintenanceType, + RedisConnectionException connection => connection.MaintenanceType, + _ => MaintenanceNotificationType.None, + }; + Console.WriteLine($"[command] {ex.GetType().Name}: {ex.Message}" + + (maintenance == MaintenanceNotificationType.None ? "" : $" <- during {maintenance}")); + } + + await Task.Delay(1000); +} diff --git a/toys/StackExchange.Redis.Server/RedisClient.Output.cs b/toys/StackExchange.Redis.Server/RedisClient.Output.cs index 0e9396a5a..b0a2b21d6 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.Output.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.Output.cs @@ -27,6 +27,30 @@ private readonly struct VersionedResponse(TypedRedisValue value, RedisProtocol p private readonly Channel _replies = Channel.CreateUnbounded(s_replyChannelOptions); + private TypedRedisValue _deferred = TypedRedisValue.Nil; + + /// + /// Queues a frame to be sent *after* the reply to the command currently being executed. + /// + /// + /// Needed because the read loop enqueues a command's reply once Execute has returned, so a handler + /// that calls directly puts its frame *before* its own reply. A real server does + /// the opposite - the retained maintenance notification arrives immediately after the +OK for the + /// opt-in, in the same TCP segment - and the ordering is the whole point of modelling this. + /// + public void AddOutboundAfterReply(in TypedRedisValue message) + { + FlushDeferredOutbound(); // one slot is enough for any real case; do not lose an earlier frame + _deferred = message; + } + + internal void FlushDeferredOutbound() + { + var pending = _deferred; + _deferred = TypedRedisValue.Nil; + if (!pending.IsNil) AddOutbound(pending); + } + public void AddOutbound(in TypedRedisValue message) { if (message.IsNil) @@ -139,6 +163,9 @@ public async Task WriteOutputAsync(PipeWriter writer) } catch (Exception ex) { + // this used to vanish into the pipe: a serialization bug in a fake server's *outbound* path + // presents as "the client never received it", with a reconnect covering the tracks. Log it + Node?.Server?.Log($"[{this}] write loop faulted: {ex.Message}"); await writer.CompleteAsync(ex); } } diff --git a/toys/StackExchange.Redis.Server/RedisClient.cs b/toys/StackExchange.Redis.Server/RedisClient.cs index 69fa5437a..bed7b00f0 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.cs @@ -157,7 +157,12 @@ public void Dispose() } private int _activeSlot = ServerSelectionStrategy.NoSlot; - internal void ResetAfterRequest() => _activeSlot = ServerSelectionStrategy.NoSlot; + internal void ResetAfterRequest() + { + _activeSlot = ServerSelectionStrategy.NoSlot; + FlushDeferredOutbound(); // after the reply for this command, which the read loop has now queued + } + public virtual void OnKey(in RedisKey key, KeyFlags flags) { if ((flags & KeyFlags.NoSlotCheck) == 0 & node.CheckCrossSlot) diff --git a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs index 14eda7143..51530f5a9 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.Net; +using System.Threading; +using System.Threading.Tasks; using RESPite; using RESPite.Messages; @@ -54,6 +57,96 @@ public enum MaintenanceNotificationKind }; private int _maintenanceSequence; + private int _maintenanceOptIns; + private (MaintenanceNotificationKind Kind, long Sequence, string ShardIds)? _retainedCompletion; + + /// + /// Whether the server keeps the most recent shard-scoped *completion* and replays it to each + /// connection that opts in, as Redis Enterprise does. + /// + /// + /// Observed on RS 8.0.22 (2026-08-28), and the boundary is sharp: MIGRATED and + /// FAILED_OVER are retained; MIGRATING, FAILING_OVER, MOVING, + /// SMIGRATING and SMIGRATED are not. The characterisation that fits all seven is the + /// completion of a *shard-scoped* event - the two that carry an affected-shards list - so + /// SMIGRATED (a slot-range triple) and MOVING (an endpoint) are excluded even though + /// SMIGRATED is also a completion. + /// + /// The consequence is a design property worth stating: the catch-up channel can only ever say "a + /// disruption ended", never "one is starting". Nothing that demands action is replayed, so a + /// reconnecting client cannot be told to move by a stale frame. + /// + /// + /// Retained "most recent, replaced not accumulated" - one frame, never a queue - so a connection sees + /// at most one of these however many events went past. + /// + /// + public bool RetainCompletions { get; set; } = true; + + /// + /// Whether this notification is one of the two the server retains for replay. + /// + private static bool IsRetainedCompletion(MaintenanceNotificationKind kind) + => kind is MaintenanceNotificationKind.Migrated or MaintenanceNotificationKind.FailedOver; + + /// + /// Replays the retained completion, if any, to a client that has just opted in. + /// + /// + /// Deliberately *after* the +OK (see ), and with + /// the original sequence id rather than a fresh one: the id identifies the event, and a client using it + /// for dedup has to be able to recognize a completion it has already seen. + /// + private void ReplayRetainedCompletion(RedisClient client) + { + if (!RetainCompletions || _retainedCompletion is not { } retained) return; + + client.AddOutboundAfterReply(BuildShardNotification(retained.Kind, null, retained.Sequence, null, retained.ShardIds)); + Log($"[{client}] replayed retained {GetName(retained.Kind)} (seq {retained.Sequence})"); + } + + /// + /// How many times a client has opted in, across the lifetime of the server. Per-client state goes away + /// with the client, so this is what a reconnect test can measure. + /// + public int TotalMaintenanceOptIns => Volatile.Read(ref _maintenanceOptIns); + + internal void OnMaintenanceOptIn() => Interlocked.Increment(ref _maintenanceOptIns); + + /// + /// Whether announces itself the way a real server does, rather + /// than only moving the slot in this model. + /// + /// + /// Off by default, because plenty of tests use Migrate purely to arrange a topology and would + /// not expect a notification to arrive mid-arrangement. Turn it on to exercise the *sequence* a client + /// really sees - which is the only way to test how the notification-driven path and the unsolicited + /// SUNSUBSCRIBE path interact, since both fire for the same migration. + /// + public bool NotifyOnMigrate { get; set; } + + /// + /// Announces a slot migration the way a real server does: the shard notifications either side of it, + /// and an unsolicited sunsubscribe to any client subscribed to a sharded channel that has just + /// moved away. + /// + /// + /// Note the two signals are independent. The notifications only reach clients that opted in, whereas + /// the unsubscribe is ordinary cluster behaviour and reaches every subscriber - so a client can + /// legitimately see one, the other, or both, and the order is a server implementation detail. That is + /// exactly the interaction worth being able to reproduce here. + /// + private void AnnounceMigration(int hashSlot, Node from, Node to) + { + var slots = hashSlot.ToString(System.Globalization.CultureInfo.InvariantCulture); + SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, slots); + + var dropped = ForAllClients(hashSlot, static (client, slot) => client.UnsubscribeMigratedSlot(slot)); + if (dropped != 0) Log($"unsubscribed {dropped} sharded subscription(s) for migrated slot {hashSlot}"); + + SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [($"{from.Host}:{from.Port}", $"{to.Host}:{to.Port}", slots)]); + } /// /// The sequence id given to the next notification, unless one is supplied explicitly. The contract does @@ -69,14 +162,97 @@ public enum MaintenanceNotificationKind /// /// The number of clients the notification was sent to. public int SendMoving(RedisClient client, int timeSeconds, EndPoint newEndpoint, int? sequenceId = null) - => Send(client, MaintenanceNotificationKind.Moving, timeSeconds, sequenceId, newEndpoint, null); + { + var recipients = MovingClosesConnection ? new List() : null; + Action onSent = recipients is null ? null : recipients.Add; + var count = Send(client, MaintenanceNotificationKind.Moving, timeSeconds, sequenceId, newEndpoint, null, onSent); + + if (recipients is { Count: > 0 }) + { + // The half of MOVING that defines it: the socket goes away - and it takes every other + // connection to the same node with it (see MovingClosesConnection). + var delay = MovingCloseDelay ?? TimeSpan.FromSeconds(Math.Max(timeSeconds, 0)) + MeasuredCloseSlack; + _ = CloseAfterAsync(CollectSiblings(recipients), delay); + } + + return count; + } + + /// + /// Whether then closes the affected connections, as a real proxy does. + /// + /// + /// Note the blast radius is the *node*, not the connection. Measured on RS (2026-08-28) with four + /// connections to one node differing only in handshake - two opted in, one RESP3 without the opt-in, + /// one RESP2 - all four closed simultaneously, and only the two that had opted in were warned. So this + /// is endpoint retirement rather than socket recycling, and a connection that did not opt in gets no + /// warning at all before it dies: an argument for opting in on every connection rather than one. + /// + public bool MovingClosesConnection { get; set; } + + /// + /// How much later than the announced window the close actually arrives, by default. + /// + /// + /// Measured at +3.4s and +1.6s against a declared 15s grace, i.e. the announced window behaves as a + /// floor with slack rather than a deadline. Tests should assert that a client acts *within* the window, + /// never that the socket survives to the end of it. + /// + public TimeSpan MeasuredCloseSlack { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// How long after MOVING the connections are closed; defaults to the announced window plus + /// , which is what a real proxy was measured doing. + /// + /// + /// Set it shorter than the announced window to exercise a proxy less generous than the one measured - + /// a client that treats the window as guaranteed rather than as a budget fails that case. It cannot + /// usefully be set to zero: the close would race the delivery of the notification itself, which is not + /// something any real timing produces. + /// + public TimeSpan? MovingCloseDelay { get; set; } + + /// + /// Expands the notified connections to every connection sharing a node with them. + /// + private List CollectSiblings(List notified) + { + var nodes = new HashSet(); + foreach (var client in notified) nodes.Add(client.Node); + + var all = new List(); + ForAllClients( + all, + (client, list) => + { + if (nodes.Contains(client.Node)) list.Add(client); + return 0; + }); + return all; + } + + private async Task CloseAfterAsync(List clients, TimeSpan delay) + { + if (delay > TimeSpan.Zero) await Task.Delay(delay).ConfigureAwait(false); + foreach (var client in clients) + { + Log($"[{client}] closing connection after MOVING"); + client.Kill(); + } + } /// /// Sends one of the shard-scoped notifications. is a remaining-time - /// delta and may legitimately be zero or negative for a connection that arrived mid-window. + /// delta and may legitimately be zero or negative for a connection that arrived mid-window; pass + /// null to omit the element entirely, which is what a real server does for the *closing* + /// notifications - captured from Enterprise 8.6.2: + /// + /// >4 $12 FAILING_OVER :0 :2 $6 ["21"] + /// >3 $11 FAILED_OVER :1 $6 ["21"] + /// /// /// The number of clients the notification was sent to. - public int SendShardNotification(RedisClient client, MaintenanceNotificationKind kind, int timeSeconds, string shardIds = null, int? sequenceId = null) + public int SendShardNotification(RedisClient client, MaintenanceNotificationKind kind, int? timeSeconds, string shardIds = null, int? sequenceId = null) => Send(client, kind, timeSeconds, sequenceId, null, shardIds); /// @@ -87,6 +263,48 @@ public int SendShardNotification(RedisClient client, MaintenanceNotificationKind public int SendSlotNotification(RedisClient client, MaintenanceNotificationKind kind, string slots, int? sequenceId = null) => Send(client, kind, null, sequenceId, null, slots); + /// + /// Sends SMIGRATED in its nested form - [type, seq, [[source, target, slots], ...]] - + /// which is what the shipped clients read (see the topic README's prior art; go-redis reads exactly + /// this shape and redis-py models the same nesting). + /// + /// + /// Note the sender is not implicitly the source of anything: every node reports the same movements, so + /// a test can and should exercise a delta that does not involve this server at all. + /// + /// The number of clients the notification was sent to. + public int SendSlotMigrations(RedisClient client, MaintenanceNotificationKind kind, (string Source, string Target, string Slots)[] migrations, int? sequenceId = null) + { + // one sequence id for the notification, however many clients it is delivered to - it identifies the + // event, not the delivery, which is what a real server does and what client-side dedup relies on + var seq = sequenceId ?? Interlocked.Increment(ref _maintenanceSequence); + return Dispatch(client, Build, requireOptIn: true); + + TypedRedisValue Build() + { + // Rent at every level: Recycle() recurses, so a Standalone child inside a pooled parent gets + // handed to the pool it did not come from ("The buffer is not associated with this pool") + var frame = TypedRedisValue.Rent(3, out var span, RespPrefix.Push); + // bulk, not simple: that is what a real server sends. Captured from Enterprise 8.6.2: + // >3 $9 SMIGRATED :19 *1[ *3[ $20 $18 $9 ] ] + span[0] = TypedRedisValue.BulkString(GetName(kind)); + span[1] = TypedRedisValue.Integer(seq); + + var outer = TypedRedisValue.Rent(migrations.Length, out var outerSpan, RespPrefix.Array); + for (int i = 0; i < migrations.Length; i++) + { + var triplet = TypedRedisValue.Rent(3, out var inner, RespPrefix.Array); + inner[0] = TypedRedisValue.BulkString(migrations[i].Source); + inner[1] = TypedRedisValue.BulkString(migrations[i].Target); + inner[2] = TypedRedisValue.BulkString(migrations[i].Slots); + outerSpan[i] = triplet; + } + + span[2] = outer; + return frame; + } + } + /// /// Sends an arbitrary push frame to a client, for the cases a well-formed notification cannot express: /// an unknown type, a malformed payload, extra trailing elements. @@ -94,12 +312,17 @@ public int SendSlotNotification(RedisClient client, MaintenanceNotificationKind /// The number of clients the frame was sent to. public int SendRawPush(RedisClient client, params string[] parts) { - var frame = TypedRedisValue.Rent(parts.Length, out var span, RespPrefix.Push); - for (int i = 0; i < parts.Length; i++) + return Dispatch(client, Build, requireOptIn: false); + + TypedRedisValue Build() { - span[i] = TypedRedisValue.BulkString(parts[i]); + var frame = TypedRedisValue.Rent(parts.Length, out var span, RespPrefix.Push); + for (int i = 0; i < parts.Length; i++) + { + span[i] = TypedRedisValue.BulkString(parts[i]); + } + return frame; } - return Dispatch(client, frame, requireOptIn: false); } private int Send( @@ -108,16 +331,35 @@ private int Send( int? timeSeconds, int? sequenceId, EndPoint newEndpoint, - string extra) + string extra, + Action onSent = null) { // [type, seqID, ...] - the sequence id is an integer, which is precisely why these frames could // not be treated as pub/sub: element 1 is not a channel name + var seq = sequenceId ?? System.Threading.Interlocked.Increment(ref _maintenanceSequence); + + // the retention is server-wide and replaces rather than accumulates, as a real one does; note it + // records what was *sent*, so a test arranges it by sending the completion, not by poking state + if (IsRetainedCompletion(kind)) _retainedCompletion = (kind, seq, extra); + + return Dispatch(client, Build, requireOptIn: true, onSent); + + TypedRedisValue Build() => BuildShardNotification(kind, timeSeconds, seq, newEndpoint, extra); + } + + private static TypedRedisValue BuildShardNotification( + MaintenanceNotificationKind kind, + int? timeSeconds, + long seq, + EndPoint newEndpoint, + string extra) + { int count = 2 + (timeSeconds.HasValue ? 1 : 0) + (newEndpoint is not null || kind == MaintenanceNotificationKind.Moving ? 1 : 0) + (extra is not null ? 1 : 0); var frame = TypedRedisValue.Rent(count, out var span, RespPrefix.Push); int index = 0; - span[index++] = TypedRedisValue.SimpleString(GetName(kind)); - span[index++] = TypedRedisValue.Integer(sequenceId ?? System.Threading.Interlocked.Increment(ref _maintenanceSequence)); + span[index++] = TypedRedisValue.BulkString(GetName(kind)); // bulk, as a real server sends + span[index++] = TypedRedisValue.Integer(seq); if (timeSeconds.HasValue) span[index++] = TypedRedisValue.Integer(timeSeconds.GetValueOrDefault()); if (kind == MaintenanceNotificationKind.Moving) { @@ -132,27 +374,38 @@ private int Send( } if (extra is not null) span[index] = TypedRedisValue.BulkString(extra); - return Dispatch(client, frame, requireOptIn: true); + return frame; } - private int Dispatch(RedisClient client, in TypedRedisValue frame, bool requireOptIn) + /// + /// Sends a notification to one client or to every opted-in client, building the frame per recipient. + /// + /// + /// The factory is called once per recipient rather than the frame being built once and shared, because + /// each client's write loop recycles what it wrote: a shared frame is returned to the pool by the first + /// writer and the second one faults with "Array element cannot be nil", killing that connection. Every + /// broadcast therefore used to reach exactly one client, which is invisible in a single-node test and + /// silently made multi-node fan-out untestable. + /// + private int Dispatch(RedisClient client, Func frameFactory, bool requireOptIn, Action onSent = null) { if (client is not null) { - client.AddOutbound(frame); + client.AddOutbound(frameFactory()); + onSent?.Invoke(client); return 1; } // a real server sends only to connections that asked, so sending to all means all *opted-in*. // Counting the sends rather than the clients visited: the Action overload of ForAllClients returns // one per client regardless, which would report every connection as a recipient - var copy = frame; return ForAllClients( requireOptIn, (target, gated) => { if (gated && !target.MaintenanceNotifications) return 0; - target.AddOutbound(copy); + target.AddOutbound(frameFactory()); + onSent?.Invoke(target); return 1; }); } diff --git a/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs b/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs index 7778ed63b..1498ce1c0 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs @@ -62,7 +62,9 @@ protected virtual TypedRedisValue SPublish(RedisClient client, in RedisRequest r var slot = ServerSelectionStrategy.GetClusterSlot((byte[])channel); if (!node.HasSlot(slot)) KeyMovedException.Throw(slot); - PublishPair pair = new(channel, request.GetValue(2)); + // note the node: without it pair.Node is null, ReferenceEquals never matches, and sharded publish + // silently delivers to nobody - which is how it behaved until 2026-08-26 + PublishPair pair = new(channel, request.GetValue(2), node); int count = ForAllClients(pair, static (client, pair) => ReferenceEquals(client.Node, pair.Node) ? client.Publish(pair.Channel, pair.Value) : 0); return TypedRedisValue.Integer(count); @@ -280,6 +282,38 @@ private Regex BuildGlob(RedisChannel channel) return new Regex(re, RegexOptions.CultureInvariant); } + /// + /// Drops this client's sharded subscriptions whose channel hashes into , + /// pushing the unsolicited sunsubscribe a real server sends when a slot moves away. + /// + /// How many subscriptions were dropped. + internal int UnsubscribeMigratedSlot(int hashSlot) + { + var subs = SubscriptionsIfAny; + if (subs is null) return 0; + + List affected = null; + lock (subs) + { + foreach (var pair in subs) + { + var channel = pair.Key; + if (channel.IsSharded && ServerSelectionStrategy.GetClusterSlot((byte[])channel) == hashSlot) + { + (affected ??= new()).Add(channel); + } + } + } + + if (affected is null) return 0; + foreach (var channel in affected) + { + Unsubscribe(channel); // removes it *and* sends the push, exactly as a client-issued one would + } + + return affected.Count; + } + internal void Unsubscribe(RedisChannel channel) { var subs = SubscriptionsIfAny; diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index af2bd890d..88d20a262 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -112,6 +112,12 @@ public bool Migrate(int hashSlot, EndPoint to) throw new KeyNotFoundException($"Unable to remove slot {hashSlot} from old owner"); } target.AddSlot(hashSlot); + + if (NotifyOnMigrate) + { + AnnounceMigration(hashSlot, pair.Value, target); + } + return true; } } @@ -311,6 +317,38 @@ public EndPoint AddEmptyNode(EndPoint endpoint, NodeFlags flags = NodeFlags.None return endpoint; } + /// + /// Removes a node from the cluster entirely, as CLUSTER FORGET does: it stops appearing in + /// CLUSTER SLOTS and in CLUSTER NODES, and any name it answered to stops resolving. + /// + /// + /// Refuses while it still owns slots, which is also what a real cluster does - a node has to have its + /// slots migrated away before it can be forgotten, and allowing it here would produce a topology with + /// unowned slots that says nothing useful about client behaviour. + /// + /// This exists to distinguish the two conditions a client has to tell apart: a node that currently + /// serves nothing is still a cluster member and may be given slots again, whereas a node that has + /// *left* is one whose connection should be given up. + /// + /// + public bool RemoveNode(EndPoint endpoint) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (!_nodes.TryGetValue(endpoint, out var node)) return false; + if (node.HasAnySlot) throw new InvalidOperationException($"Node still owns slots: {Format.ToString(endpoint)}"); + + if (!_nodes.TryRemove(endpoint, out _)) return false; + + // and every alias that pointed at it, or a stale name would resolve to a node that is gone + foreach (var pair in _aliases) + { + if (ReferenceEquals(pair.Value, node)) _aliases.TryRemove(pair.Key, out _); + } + + Log($"node removed from the cluster: {Format.ToString(endpoint)}"); + return true; + } + public EndPoint AddEmptyNode(NodeFlags flags = NodeFlags.None) { EndPoint endpoint; @@ -722,6 +760,9 @@ public enum MaintenanceNotificationSupport // explicitly valid and means "use the server defaults", so the parameter list is optional and // unrecognized parameters are an error rather than something to ignore - the client is asking the // server to do something specific, and silently not doing it would be worse than refusing + private static bool IsKeyword(in RedisRequest request, int index, string keyword) + => string.Equals(request.GetString(index), keyword, StringComparison.OrdinalIgnoreCase); + [RedisCommand(-3, nameof(RedisCommand.CLIENT), "maint_notifications", LockFree = true)] protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, in RedisRequest request) { @@ -733,9 +774,12 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i return TypedRedisValue.Error("ERR maintenance notifications are disabled on this server"); } + // keywords are matched case-insensitively, as a real server does; clients differ here (go-redis + // sends lowercase, we send the same uppercase form we use for every other keyword), and a fake + // that only accepted one of them would fail a client for something a real server allows bool on; - if (request.IsString(2, "on"u8)) on = true; - else if (request.IsString(2, "off"u8)) on = false; + if (IsKeyword(request, 2, "on")) on = true; + else if (IsKeyword(request, 2, "off")) on = false; else return TypedRedisValue.Error("ERR syntax error"); string movingEndpointType = null; @@ -743,9 +787,9 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i { if (i + 1 >= request.Count) return TypedRedisValue.Error("ERR syntax error"); - if (request.IsString(i, "moving-endpoint-type"u8)) + if (IsKeyword(request, i, "moving-endpoint-type")) { - movingEndpointType = request.GetString(i + 1); + movingEndpointType = request.GetString(i + 1)?.ToLowerInvariant(); switch (movingEndpointType) { case "internal-ip": @@ -767,6 +811,11 @@ protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, i client.MaintenanceNotifications = on; client.MovingEndpointType = on ? movingEndpointType : null; client.MaintenanceNotificationOptInCount++; + if (on) + { + OnMaintenanceOptIn(); + ReplayRetainedCompletion(client); // the catch-up channel, immediately after this +OK + } Log($"[{client}] maintenance notifications {(on ? "on" : "off")}, moving-endpoint-type: {movingEndpointType ?? "(server default)"}"); return TypedRedisValue.OK; } @@ -1080,6 +1129,12 @@ public Node(RedisServer server, EndPoint endpoint, NodeFlags flags) } public void UpdateSlots(SlotRange[] slots) => _slots = slots; + + /// + /// Whether this node serves anything at all; note a null slot set means "all slots" + /// (the single-node default), not "none". + /// + public bool HasAnySlot => _slots is null || _slots.Length != 0; public ReadOnlySpan Slots => _slots ?? SlotRange.SharedAllSlots; public bool CheckCrossSlot => _server.CheckCrossSlot;