Maintenance notifications: opt in, and receive (D2)

Directory.Build.props

@@ -11,7 +11,7 @@
     <CodeAnalysisRuleset>$(MSBuildThisFileDirectory)Shared.ruleset</CodeAnalysisRuleset>
     <MSBuildWarningsAsMessages>NETSDK1069</MSBuildWarningsAsMessages>
     <!-- SER002/SER003/SER006 retired (Redis 8.4/8.6/8.8 features no longer experimental); IDs reserved, see Experiments.cs -->
-    <NoWarn>$(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009</NoWarn>
+    <NoWarn>$(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009;SER010</NoWarn>
     <PackageReleaseNotes>https://github.com/StackExchange/StackExchange.Redis/releases</PackageReleaseNotes>
     <PackageProjectUrl>https://seredis.dev/</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>

docs/Configuration.md

@@ -100,6 +100,11 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a
 | setlib={bool}          | `SetClientLibrary`     | `true`                       | Whether to attempt to use `CLIENT SETINFO` to set the library name/version on the connection              |
 | protocol={string}      | `Protocol`             | `null`                       | Redis protocol to use; see section below                                                                  |
 | highIntegrity={bool}   | `HighIntegrity`        | `false`                      | High integrity (incurs overhead) sequence checking on every command; see section below                    |
+| defaults={string}      | `Defaults`             | `null`                       | Selects a named defaults provider; see section below                                                      |
+| maintNotifications={string} | `MaintenanceNotifications` | `disabled`           | Whether to ask servers for maintenance notifications; see section below                                    |
+| maintRelaxedTimeout={int}   | `MaintenanceRelaxedTimeout` | `10`                | **Seconds** that command timeouts are relaxed to during announced maintenance; see section below           |
+| maintRelaxedWindowMax={int} | `MaintenanceRelaxedWindowMax` | `30`              | **Seconds** a relaxed window may last at most; see section below                                          |
+| maintPostEventRelaxed={int} | `MaintenancePostEventRelaxedDuration` | `20`      | **Seconds** timeouts stay relaxed after maintenance completes; see section below                           |
 
 Additional code-only options:
 - LoggerFactory (`ILoggerFactory`) - Default: `null`
@@ -287,6 +292,57 @@ config.ReconnectRetryPolicy = new LinearRetry(5000);
 //6	        5000
 ```
 
+## Defaults providers
+
+Some settings have sensible values that depend on *what you are connecting to* rather than on what you want, so
+the library keeps them in a provider and consults it for anything you have not set explicitly. Providers are
+normally chosen automatically by looking at the endpoints - an `*.redis.cache.windows.net` host selects the
+Azure provider, and so on - and you can select one explicitly instead:
+
+```
+myserver:6379,defaults=enterprise
+```
+
+The names are `azure`, `amr` (Azure Managed Redis), `rediscloud` and `enterprise` (a self-managed Redis
+Enterprise deployment). The last one exists because it cannot be detected: a self-managed cluster has whatever
+DNS its operator gave it, so there is nothing to recognize. It is also the right choice for a hosted deployment
+reached behind private DNS, a CNAME or a proxy, where the endpoint no longer looks like what it is.
+
+A provider chosen explicitly appears in `ToString()`; one that was merely inferred from the endpoints does not,
+since writing it out would turn a guess into a decision. Custom providers (assigned in code, via
+`ConfigurationOptions.Defaults`, or registered with `DefaultOptionsProvider.AddProvider`) work exactly as
+before; they can additionally be named in a configuration string if they override `Name`.
+
+## Maintenance notifications
+
+Redis Enterprise and Redis Cloud can warn a connected client *before* a disruptive event - a shard migration, a
+failover, or an endpoint being replaced - so the client can act ahead of it rather than discover it by way of a
+broken connection. This requires RESP3, and the client asks for it per connection:
+
+| Mode | Behaviour |
+| --- | --- |
+| `disabled` | Never ask. The default, because most servers have never heard of the request. |
+| `auto` | Ask, and carry on if the server refuses or the connection ends up RESP2. Safe against a mixture of servers, and what most callers want. |
+| `enabled` | **Required**: ask, and reject the connection unless notifications are live. Only point this at a deployment you know supports them. |
+
+Note that `enabled` means *required*, not merely *on* - it is the cross-client name for that mode, and it will
+refuse a connection that cannot deliver notifications, including any RESP2 connection. The `amr`, `rediscloud`
+and `enterprise` providers select `auto` for you, so on those deployments you need not set anything.
+
+While a disruption has been announced, command timeouts are relaxed - raised to `maintRelaxedTimeout`, never
+lowered, so a caller with a more generous timeout keeps it. The window ends when the server says the disruption
+finished, and then stays relaxed for `maintPostEventRelaxed` longer, because completion is exactly when every
+other client that received the same notification comes back. `maintRelaxedWindowMax` bounds a window whose
+closing notification never arrives. Only *command* timeouts are affected: keep-alive and connection-failure
+detection are untouched, so a server that dies mid-maintenance is still noticed on the usual schedule.
+
+These durations are in **seconds**, unlike every other timeout here, because those are the units the
+cross-client specification names for them.
+
+Receiving a notification raises `ConnectionMultiplexer.ServerMaintenanceEvent` with a `PushMaintenanceEvent`,
+and a timeout or connection fault that happened during an announced disruption carries a `MaintenanceType`
+saying so.
+
 ## Redis protocol
 
 RESP3 is a newer protocol (available on v6 servers and above) which allows (among other changes) pub/sub messages to be communicated on the *same* connection - which can be very

docs/exp/SER010.md

@@ -0,0 +1,58 @@
+Maintenance notifications ("smart client handoffs") are a server feature by which a server warns a
+connected client in advance of a disruptive event - a shard migration, a failover, or the endpoint
+itself being replaced - so the client can act before the disruption rather than react to a broken
+connection afterwards. The client opts in per connection with `CLIENT MAINT_NOTIFICATIONS ON`, and the
+notifications then arrive as RESP3 push frames on the connection that carries commands.
+
+The feature is experimental here for three separate reasons:
+
+1. **The server side is not universal.** Only Redis Enterprise and Redis Cloud emit these
+   notifications today; OSS Redis, Valkey and Garnet do not recognize the opt-in at all. That is why
+   `MaintenanceNotificationMode.Auto` exists, and why the default is `Disabled`.
+2. **The wire contract is still moving.** Notification types have been proposed and withdrawn during
+   its development, the semantics of some fields (notably the sequence number) are not defined by any
+   specification, and no captured RESP transcript exists to validate against. Our parsing is
+   deliberately liberal, but the shapes may change.
+3. **What the client *does* in response is the substantial part**, and it is being built in stages -
+   timeout relaxation, then endpoint handoff. Behaviour may therefore change materially between
+   versions while the diagnostic is in place, even where the API does not.
+
+`MaintenanceNotificationMode`:
+
+- `Disabled` (the default) - never ask; the server sends nothing.
+- `Enabled` - **required**: ask, and *reject the connection* unless notifications end up live. That covers a
+  server that refuses, a server that answers `HELLO 3` as RESP2, and a configuration that could never ask in
+  the first place (`Protocol = Resp2`, or `HELLO` unavailable) - requiring a RESP3-only feature over RESP2 is
+  a contradiction, and it fails rather than being half-honoured. Only point this at a deployment you know
+  supports them.
+- `Auto` - ask, and carry on if the server refuses or the connection ends up RESP2; the feature is then
+  simply off for that server and the connection is never rejected over it. Safe against a mixture of
+  servers, or one whose support you don't know, and what most callers want.
+
+The names are the cross-client ones (they match go-redis and redis-py, so a connection string ports
+between clients), which makes `Enabled` easy to misread as "on" - it means "required". One deliberate
+divergence: the cross-client spec only calls for interrupting the connection when the *server* errors,
+whereas we extend that to the RESP2 cases above, on the grounds that a required feature which cannot
+possibly be delivered is the same failure either way.
+
+## A deployment prerequisite worth knowing
+
+On Redis Enterprise there are *two* switches, and they are easy to conflate. A **cluster-level flag** decides
+whether the subcommand exists at all - `client_maint_notifications` for proxy-routed databases,
+`oss_cluster_client_maint_notifications` for `oss_cluster` ones - and the **per-connection opt-in** this option
+controls asks one connection to receive them. A cluster running a supporting version with the flag off will
+refuse `CLIENT MAINT_NOTIFICATIONS ON`, which `Auto` absorbs silently and `Enabled` turns into a refused
+connection. So if `Enabled` is rejecting connections against a deployment you believe supports the feature,
+check the cluster flag before suspecting the client.
+
+If you accept the above, you can suppress this warning by adding the following to your `csproj` file:
+
+```xml
+<NoWarn>$(NoWarn);SER010</NoWarn>
+```
+
+or more granularly / locally in C#:
+
+``` c#
+#pragma warning disable SER010
+```

src/RESPite/Shared/Experiments.cs

@@ -21,6 +21,7 @@ internal static class Experiments
         public const string GeoRedundantFailover = "SER007";
         public const string Server_8_10 = "SER008";
         public const string Transport = "SER009";
+        public const string MaintenanceNotifications = "SER010";
 
         // ReSharper restore InconsistentNaming
 

src/StackExchange.Redis/Availability/FaultContext.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Diagnostics.CodeAnalysis;
 using RESPite;
 
@@ -39,11 +39,13 @@ public FaultContext(Exception fault)
                 kind = RedisErrorKind.ConnectionFault;
                 flags = connection.Flags;
                 status = connection.CommandStatus;
+                MaintenanceType = connection.MaintenanceType;
                 break;
             case RedisTimeoutException timeout:
                 kind = RedisErrorKind.Timeout;
                 flags = timeout.Flags;
                 status = timeout.Commandstatus;
+                MaintenanceType = timeout.MaintenanceType;
                 break;
             case TimeoutException:
                 kind = RedisErrorKind.Timeout;
@@ -118,6 +120,18 @@ public FaultContext(Exception fault)
     /// </summary>
     public ConnectionFailureType ConnectionFailureType => _connectionFailureType;
 
+    /// <summary>
+    /// The maintenance notification in force when this fault happened, if any.
+    /// </summary>
+    /// <remarks>
+    /// Useful to a circuit breaker: a fault during announced maintenance is expected and transient, and
+    /// counting it towards a trip that then withdraws a whole server is the opposite of what the notification
+    /// was for. Left to policy rather than acted on here, since "ignore faults during maintenance" is a
+    /// judgement about the deployment, not about the protocol.
+    /// </remarks>
+    [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)]
+    public Maintenance.MaintenanceNotificationType MaintenanceType { get; }
+
     private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status)
     {
         // the client never handed it to the socket, so the server cannot have seen it

src/StackExchange.Redis/Availability/HealthCheckContext.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Diagnostics.CodeAnalysis;
 using RESPite;
 
@@ -27,4 +27,16 @@ public readonly struct HealthCheckContext(IServer server, TimeSpan probeTimeout)
     /// themselves (the caller applies it), but may use it to bound any state they create.
     /// </summary>
     public TimeSpan ProbeTimeout => probeTimeout;
+
+    /// <summary>
+    /// Gets flags that a probe should include on every command it issues.
+    /// </summary>
+    /// <remarks>
+    /// This marks the traffic as a health check rather than as work a caller is waiting on. It matters because
+    /// idleness is what decides whether an endpoint can be given up: a probe that looks like caller work makes
+    /// a server appear busy precisely because we are watching it, and an endpoint that has left the deployment
+    /// then never gets retired. The built-in probes pass this; a custom probe should too, and passing it
+    /// changes nothing else about how the command is routed or queued.
+    /// </remarks>
+    public CommandFlags ProbeFlags => Message.ProbeFlag;
 }

src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs

@@ -1,4 +1,4 @@
-using System.Threading.Tasks;
+using System.Threading.Tasks;
 
 namespace StackExchange.Redis.Availability;
 
@@ -16,7 +16,7 @@ private PingProbe() { }
 
         public override async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context)
         {
-            await context.Server.PingAsync();
+            await context.Server.PingAsync(context.ProbeFlags);
             return HealthCheckResult.Healthy;
         }
     }

src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Buffers;
 using System.Threading.Tasks;
 
@@ -39,10 +39,10 @@ await database.LockTakeAsync(
                     key: key,
                     value: payload,
                     expiry: context.ProbeTimeout,
-                    flags: CommandFlags.FireAndForget).ForAwait();
+                    flags: CommandFlags.FireAndForget | context.ProbeFlags).ForAwait();
 
                 // release from the db if matches (otherwise, we have no clue what happened, so: leave alone)
-                var success = await database.LockReleaseAsync(key, payload).ForAwait();
+                var success = await database.LockReleaseAsync(key, payload, context.ProbeFlags).ForAwait();
                 return success ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy;
             }
             finally

src/StackExchange.Redis/ClusterConfiguration.cs

@@ -137,21 +137,93 @@ public override int GetHashCode()
 
         internal bool Includes(int hashSlot) => hashSlot >= from && hashSlot <= to;
 
-        private static bool TryParseInt16(string s, int offset, int count, out short value)
+        /// <summary>
+        /// Parses one range from within a larger string, so that a comma-separated list can be read without
+        /// allocating a substring per element.
+        /// </summary>
+        internal static bool TryParse(string value, int offset, int length, out SlotRange range)
         {
-            checked
+            range = default;
+            if (length <= 0) return false;
+
+            int dash = -1;
+            for (int i = 0; i < length; i++)
+            {
+                if (value[offset + i] == '-')
+                {
+                    dash = i;
+                    break;
+                }
+            }
+
+            if (dash < 0)
             {
-                value = 0;
-                int tmp = 0;
-                for (int i = 0; i < count; i++)
+                if (TryParseInt16(value, offset, length, out var only))
                 {
-                    char c = s[offset + i];
-                    if (c < '0' || c > '9') return false;
-                    tmp = (tmp * 10) + (c - '0');
+                    range = new SlotRange(only, only);
+                    return true;
                 }
-                value = (short)tmp;
+                return false;
+            }
+
+            if (dash != 0 && dash != length - 1
+                && TryParseInt16(value, offset, dash, out var from)
+                && TryParseInt16(value, offset + dash + 1, length - dash - 1, out var to))
+            {
+                // a reversed range is a server bug, not something to normalize silently
+                if (from > to) return false;
+                range = new SlotRange(from, to);
                 return true;
             }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Parses the comma-and-range slot form used by the maintenance notifications, e.g.
+        /// <c>123,456,789-1000</c>. Empty elements are skipped; a malformed element fails the whole list,
+        /// since a partially-read slot set is worse than none.
+        /// </summary>
+        internal static bool TryParseList(string? value, out List<SlotRange> ranges)
+        {
+            ranges = [];
+            if (string.IsNullOrEmpty(value)) return false;
+
+            int start = 0;
+            while (start <= value!.Length)
+            {
+                var comma = value.IndexOf(',', start);
+                var length = (comma < 0 ? value.Length : comma) - start;
+                if (length > 0)
+                {
+                    if (!TryParse(value, start, length, out var range)) return false;
+                    ranges.Add(range);
+                }
+
+                if (comma < 0) break;
+                start = comma + 1;
+            }
+
+            return ranges.Count != 0;
+        }
+
+        private static bool TryParseInt16(string s, int offset, int count, out short value)
+        {
+            // note this deliberately does not use `checked`: it used to, and an out-of-range slot number
+            // therefore threw OverflowException from a Try* method - reachable from the public
+            // SlotRange.TryParse and from CLUSTER NODES parsing, and now also from the maintenance
+            // notification reader, where throwing on the read loop is far worse than rejecting a value
+            value = 0;
+            int tmp = 0;
+            for (int i = 0; i < count; i++)
+            {
+                char c = s[offset + i];
+                if (c < '0' || c > '9') return false;
+                tmp = (tmp * 10) + (c - '0');
+                if (tmp > short.MaxValue) return false; // as soon as it cannot fit, stop
+            }
+            value = (short)tmp;
+            return true;
         }
 
         int IComparable.CompareTo(object? obj) => obj is SlotRange sRange ? CompareTo(sRange) : -1;

src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Diagnostics.CodeAnalysis;
 using System.Net;
 using System.Threading.Tasks;
+using RESPite;
 
 namespace StackExchange.Redis.Configuration
 {
@@ -29,6 +31,9 @@ public class AzureManagedRedisOptionsProvider : DefaultOptionsProvider
             ".redisenterprise.cache.azure.net",
         ];
 
+        /// <inheritdoc/>
+        public override string Name => "amr";
+
         /// <inheritdoc/>
         public override bool IsMatch(EndPoint endpoint)
         {
@@ -65,5 +70,18 @@ public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action<strin
 
         /// <inheritdoc/>
         public override string ConfigurationChannel => ""; // disable on AMR
+
+        /// <summary>
+        /// Ask for maintenance notifications, tolerating a server that doesn't offer them.
+        /// </summary>
+        /// <remarks>
+        /// Pre-emptive: AMR does not emit these yet, and support is being added concurrently with this
+        /// client-side work. <see cref="MaintenanceNotificationMode.Auto"/> is what makes that safe - until
+        /// the server side ships, the opt-in is refused and the feature stays off, and it then starts working
+        /// without anybody needing to change a connection string. AMR also already prefers RESP3 here, which
+        /// the feature requires.
+        /// </remarks>
+        [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)]
+        public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto;
     }
 }