diff --git a/Directory.Build.props b/Directory.Build.props index f5981dc8b..c0863db83 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,7 +11,7 @@ $(MSBuildThisFileDirectory)Shared.ruleset NETSDK1069 - $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009 + $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008;SER009;SER010 https://github.com/StackExchange/StackExchange.Redis/releases https://seredis.dev/ MIT diff --git a/StackExchange.Redis.slnx b/StackExchange.Redis.slnx index 8b5b222aa..f3a6ac32f 100644 --- a/StackExchange.Redis.slnx +++ b/StackExchange.Redis.slnx @@ -40,6 +40,7 @@ + diff --git a/docs/Configuration.md b/docs/Configuration.md index 66eaa20ab..cc5789aa8 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -77,7 +77,8 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | connectRetry={int} | `ConnectRetry` | `3` | The number of times to repeat connect attempts during initial `Connect` | | connectTimeout={int} | `ConnectTimeout` | `5000` | Timeout (ms) for connect operations | | configChannel={string} | `ConfigurationChannel` | `__Booksleeve_MasterChanged` | Broadcast channel name for communicating configuration changes | -| configCheckSeconds={int} | `ConfigCheckSeconds` | `60` | Time (seconds) to check configuration. This serves as a keep-alive for interactive sockets, if it is supported. | +| configCheckSeconds={int} | `ConfigCheckSeconds` | `60` | Time (seconds) between re-checks of each connected server's replication role, via `INFO replication`; also acts as a keep-alive for interactive sockets. Not a topology re-read: see `topologyRefreshSeconds` | +| topologyRefreshSeconds={int} | `TopologyRefreshSeconds` | `1800` | Time (seconds) between unprompted topology re-reads, or `0` to never do so. Jittered by up to 30 seconds. | | defaultDatabase={int} | `DefaultDatabase` | `null` | Default database index, from `0` to `databases - 1` | | keepAlive={int} | `KeepAlive` | `-1` | Time (seconds) at which to send a message to help keep sockets alive (60 sec default) | | tcpKeepAlive={bool} | `TcpKeepAlive` | `true` | Enables TCP keep-alive when appropriate (endpoint- and platform-dependent) | @@ -100,6 +101,11 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | setlib={bool} | `SetClientLibrary` | `true` | Whether to attempt to use `CLIENT SETINFO` to set the library name/version on the connection | | protocol={string} | `Protocol` | `null` | Redis protocol to use; see section below | | highIntegrity={bool} | `HighIntegrity` | `false` | High integrity (incurs overhead) sequence checking on every command; see section below | +| defaults={string} | `Defaults` | `null` | Selects a named defaults provider; see section below | +| maintNotifications={string} | `MaintenanceNotifications` | `disabled` | Whether to ask servers for maintenance notifications; see section below | +| maintRelaxedTimeout={int} | `MaintenanceRelaxedTimeout` | `10` | **Seconds** that command timeouts are relaxed to during announced maintenance; see section below | +| maintRelaxedWindowMax={int} | `MaintenanceRelaxedWindowMax` | `30` | **Seconds** a relaxed window may last at most; see section below | +| maintPostEventRelaxed={int} | `MaintenancePostEventRelaxedDuration` | `20` | **Seconds** timeouts stay relaxed after maintenance completes; see section below | Additional code-only options: - LoggerFactory (`ILoggerFactory`) - Default: `null` @@ -260,6 +266,45 @@ Both options can be customized or disabled (set to `""`), via the `.Configuratio These settings are also used by the `IServer.MakeMaster()` method, which can set the tie-breaker in the database and broadcast the configuration change message. The configuration message can also be used separately to primary/replica changes simply to request all nodes to refresh their configurations, via the `ConnectionMultiplexer.PublishReconfigure` method. +## Refreshing the topology after repeated connect failures + +An endpoint that refuses every connection is evidence that what the client believes about the deployment may +be wrong, so after **three consecutive** failed connection attempts to the same endpoint, the client re-reads +the topology - the same refresh a `MOVED` or a configuration announcement would have caused, jittered and +coalesced in the same way. + +This closes a real gap rather than a theoretical one. Every other path that re-reads the topology needs +somebody *else* to notice first: a redirect from a reachable node, a peer's configuration announcement, or a +maintenance notification. The internal flag that drives a refresh-on-failure is only set once a connection has +been *established*, so an endpoint that has never connected - because it was replaced while the client was +running, or was already gone at startup - could be retried indefinitely with nobody to say otherwise. Measured +in the field: a client dialled three removed nodes for around 37 hours. + +The re-read is rate-limited to `configCheckSeconds` (default 60), deliberately reusing the knob that already +means "how often may we re-read configuration" rather than adding one. That restraint is what makes it safe: +a permanently dead endpoint, times a retry loop, times every client in a fleet would otherwise be a great +many topology reads, so a dead endpoint prompts at most one re-read per interval until something changes. + +Note that `configCheckSeconds` on its own is *not* a periodic topology refresh - it drives an +`INFO replication` on an established connection, which is a replication-role check. This is the path that +notices an endpoint nobody can reach. + +### ...and the backstop for one nobody can fault + +Repeated connect failures cover an endpoint that refuses or never finishes a handshake. What they cannot cover +is an endpoint that is *reachable*, answers a handshake, and is no longer part of the deployment: a re-bound +port now serving something else produces no failure, no redirect, and nothing announced, so no event-driven +path asks the question. + +`topologyRefreshSeconds` is the answer to that, and only that: every 30 minutes by default, the client +re-reads the topology whether or not anything appears to be wrong. Two things keep it cheap. The interval is +long, and each client picks its own phase within a 30-second jitter on every cycle, so a fleet started +together does not stay in step. Set it to `0` to turn it off. + +It is deliberately a backstop rather than the mechanism. Topology is normally learned from something +happening - a redirect, an announcement, a maintenance notification, a connection failing - and those react in +seconds where this reacts in minutes. + ## ReconnectRetryPolicy StackExchange.Redis automatically tries to reconnect in the background when the connection is lost for any reason. It keeps retrying until the connection has been restored. It would use ReconnectRetryPolicy to decide how long it should wait between the retries. @@ -287,6 +332,59 @@ config.ReconnectRetryPolicy = new LinearRetry(5000); //6 5000 ``` +## Defaults providers + +Some settings have sensible values that depend on *what you are connecting to* rather than on what you want, so +the library keeps them in a provider and consults it for anything you have not set explicitly. Providers are +normally chosen automatically by looking at the endpoints - an `*.redis.cache.windows.net` host selects the +Azure provider, and so on - and you can select one explicitly instead: + +``` +myserver:6379,defaults=enterprise +``` + +The names are `azure`, `amr` (Azure Managed Redis), `rediscloud` and `enterprise` (a self-managed Redis +Enterprise deployment). The last one exists because it cannot be detected: a self-managed cluster has whatever +DNS its operator gave it, so there is nothing to recognize. It is also the right choice for a hosted deployment +reached behind private DNS, a CNAME or a proxy, where the endpoint no longer looks like what it is. + +A provider chosen explicitly appears in `ToString()`; one that was merely inferred from the endpoints does not, +since writing it out would turn a guess into a decision. Custom providers (assigned in code, via +`ConfigurationOptions.Defaults`, or registered with `DefaultOptionsProvider.AddProvider`) work exactly as +before; they can additionally be named in a configuration string if they override `Name`. + +## Maintenance notifications + +Server-native maintenance notifications - *smart client handoffs*, also called *hitless upgrades* - are configured with the keys below; [ServerMaintenanceEvent](ServerMaintenanceEvent) is the guide to what they do. + +Redis Enterprise and Redis Cloud can warn a connected client *before* a disruptive event - a shard migration, a +failover, or an endpoint being replaced - so the client can act ahead of it rather than discover it by way of a +broken connection. This requires RESP3, and the client asks for it per connection: + +| Mode | Behaviour | +| --- | --- | +| `disabled` | Never ask. The default, because most servers have never heard of the request. | +| `auto` | Ask, and carry on if the server refuses or the connection ends up RESP2. Safe against a mixture of servers, and what most callers want. | +| `enabled` | **Required**: ask, and reject the connection unless notifications are live. Only point this at a deployment you know supports them. | + +Note that `enabled` means *required*, not merely *on* - it is the cross-client name for that mode, and it will +refuse a connection that cannot deliver notifications, including any RESP2 connection. The `amr`, `rediscloud` +and `enterprise` providers select `auto` for you, so on those deployments you need not set anything. + +While a disruption has been announced, command timeouts are relaxed - raised to `maintRelaxedTimeout`, never +lowered, so a caller with a more generous timeout keeps it. The window ends when the server says the disruption +finished, and then stays relaxed for `maintPostEventRelaxed` longer, because completion is exactly when every +other client that received the same notification comes back. `maintRelaxedWindowMax` bounds a window whose +closing notification never arrives. Only *command* timeouts are affected: keep-alive and connection-failure +detection are untouched, so a server that dies mid-maintenance is still noticed on the usual schedule. + +These durations are in **seconds**, unlike every other timeout here, because those are the units the +cross-client specification names for them. + +Receiving a notification raises `ConnectionMultiplexer.ServerMaintenanceEvent` with a `PushMaintenanceEvent`, +and a timeout or connection fault that happened during an announced disruption carries a `MaintenanceType` +saying so. + ## Redis protocol RESP3 is a newer protocol (available on v6 servers and above) which allows (among other changes) pub/sub messages to be communicated on the *same* connection - which can be very diff --git a/docs/Resp3.md b/docs/Resp3.md index 59d7128b5..9f851d61e 100644 --- a/docs/Resp3.md +++ b/docs/Resp3.md @@ -1,12 +1,13 @@ -# RESP3 and StackExchange.Redis +# RESP3 and StackExchange.Redis RESP2 and RESP3 are evolutions of the Redis protocol, with RESP3 existing from Redis server version 6 onwards (v7.2+ for Redis Enterprise). The main differences are: 1. RESP3 can carry out-of-band / "push" messages on a single connection, where-as RESP2 requires a separate connection for out-of-band (pub/sub) messages - this single connection can be of huge benefit in high-usage servers, as it halves the number of connections required -2. RESP3 supports *additional* out-of-band messages that cannot be expressed in RESP2, which allows advanced features such as "smart client handoffs" (a family of - server maintenance notifications) - - these features (not yet implemented in SE.Redis) allow for greater stability in complex deployments +2. RESP3 supports *additional* out-of-band messages that cannot be expressed in RESP2, which allows advanced features such as "smart client handoffs" (also called + "hitless upgrades"; a family of server maintenance notifications) + - these features allow for greater stability in complex deployments, and are implemented in SE.Redis: see + [ServerMaintenanceEvent](ServerMaintenanceEvent) - note they are RESP3-only, so a connection that ends up on RESP2 does not get them 3. RESP3 can (when appropriate) convey additional semantic meaning about returned payloads inside the same result structure - this is *mostly* relevant to client libraries that do not explicitly interpret the results before exposing to the user, so this does not directly impact SE.Redis itself, but it is relevant to consumers of SE.Redis that use Lua scripts or ad-hoc commands diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 2f4ba1c29..c253475fc 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -1,9 +1,232 @@ -# Introducing ServerMaintenanceEvents +# Introducing ServerMaintenanceEvents -StackExchange.Redis now automatically subscribes to notifications about upcoming maintenance from supported Redis providers. The ServerMaintenanceEvent on the ConnectionMultiplexer raises events in response to notifications about server maintenance, and application code can subscribe to the event to handle connection drops more gracefully during these maintenance operations. +StackExchange.Redis automatically subscribes to notifications about upcoming maintenance from supported Redis providers. The `ServerMaintenanceEvent` on the `ConnectionMultiplexer` raises events in response to them, and application code can subscribe to handle connection drops more gracefully during these operations. + +There are two sources, and they arrive by completely different routes: + +* **Redis Enterprise and Redis Cloud** send them as RESP3 *push frames* on the connection that carries your commands, and they surface as `PushMaintenanceEvent`. The client does not merely report these - it acts on them - and this is the direction the feature is going, so it is covered [first, below](#server-native-maintenance-notifications-redis-enterprise-and-redis-cloud). +* **Azure Cache for Redis** publishes them on a pub/sub channel (`AzureRedisEvents`), and they surface as `AzureMaintenanceEvent`. This is the original support, and is described [further down](#azure-cache-for-redis-maintenance-events-pubsub). + +Both raise the same `ServerMaintenanceEvent` event, so a handler can watch for either. If you are a Redis vendor and want to integrate support for ServerMaintenanceEvents into StackExchange.Redis, we recommend opening an issue so we can discuss the details. +# Server-native maintenance notifications (Redis Enterprise and Redis Cloud) + +> These APIs are experimental, behind diagnostic id `SER010`; see [SER010](exp/SER010.md). + +Redis Enterprise and Redis Cloud can tell a client *directly* that a disruption is coming: a shard is migrating, a node is failing over, or the endpoint you are connected to is being replaced. These arrive as RESP3 push frames on the connection itself, and the client does not merely report them: it relaxes timeouts for the duration, re-reads the cluster topology when slots have moved, recovers sharded subscriptions that were stranded, and moves off an endpoint that is going away rather than waiting to be disconnected. + +### What this feature is called + +One feature, several names, which matters mostly when you are searching: + +* **Smart client handoffs** is the name used across Redis's client libraries for the cross-client contract this implements - for example node-redis has a `smart-client-handoffs` end-to-end suite. +* **Hitless upgrades** is the same thing named after its purpose: an upgrade or rollout that does not drop the caller's work. Lettuce and redis-py both use this wording for their coverage of it, and the Redis test infrastructure treats "hitless upgrade" and "smart client handoff" as synonyms. +* **Maintenance notifications** is the name of the mechanism, and what this library calls it - go-redis names its module `maintnotifications`, and Jedis calls them maintenance events. + +Server-side you may also see it discussed as *maintenance mode*, *shard migration* and *endpoint rebinding*, which are the operations that emit the notifications rather than names for the feature. + +## Do I need to configure anything? + +Usually not. If you connect using the hostname your provider gave you, the matching options provider recognizes it and turns the feature on for you. + +| You connect to | Recognized as | Notifications | +|---|---|---| +| `something.cloud.redislabs.com`, `.cloud.redis.io`, `.redislabs.com` | Redis Cloud | on (`Auto`) | +| `something.redis.azure.net`, `.redisenterprise.cache.azure.net` | Azure Managed Redis | on (`Auto`) | +| your own hostname, a CNAME, private DNS, or through a proxy | nothing | **off** | +| a self-managed Redis Enterprise cluster | nothing (there is no DNS pattern to recognize) | **off** | + +The last two rows are the ones to know about, because nothing fails: the connection works normally and you simply never receive a notification. If your endpoint does not look like your provider's, say so explicitly. Either: + +```csharp +// the whole deployment posture: prefer RESP3, skip the OSS config-broadcast channel, and ask for notifications +var options = ConfigurationOptions.Parse("my-redis.internal.example.com:6379,defaults=enterprise"); +``` + +or, to change nothing except this feature: + +```csharp +var options = ConfigurationOptions.Parse("my-redis.internal.example.com:6379,maintNotifications=Auto"); +``` + +`defaults=` accepts `rediscloud`, `enterprise`, `amr` and `azure`; see [Configuration](Configuration.md) for what each provider sets. It is also the right answer for a *hosted* deployment reached somewhere its own provider cannot see it, such as behind a CNAME or a private endpoint. + +### RESP3 is required, and is already the default + +These notifications are RESP3 push frames, so RESP3 is a hard requirement. You do not normally need to ask for it: with no protocol configured the client assumes a 6.0 server and negotiates RESP3, which is enough. But three settings take RESP3 away again, and each one silently disables this feature: + +* `protocol=resp2` (or `Protocol = RedisProtocol.Resp2`) +* `defaultVersion` below 6.0, which is how the client decides RESP3 is available at all +* disabling or renaming `HELLO` in the [command map](Configuration.md), since RESP3 is negotiated by `HELLO` + +If you use `maintNotifications=Enabled` (see below) you will find out about this immediately, because the connection will be refused rather than quietly running without the feature. + +## Choosing a mode + +```csharp +options.MaintenanceNotifications = MaintenanceNotificationMode.Auto; +``` + +| Mode | Meaning | +|---|---| +| `Disabled` | never ask. The default when nothing recognizes your endpoint | +| `Auto` | ask, and carry on if the server says no. What the providers select | +| `Enabled` | **require** them: if the server will not deliver them, or the connection ends up on RESP2, the connection is **rejected** | + +`Auto` is the right choice almost always: asking costs one command during the handshake, and a server that accepts and then never sends anything costs nothing at all. `Enabled` exists for the case where running without advance warning is worse than not running: it turns a silent absence into a startup failure, which also makes it a useful way to prove the feature is live in a staging environment. + +### Asking where to go next + +When an endpoint is being replaced, the server can name its replacement - but only if asked. The client asks by +default (`maintMovingEndpointType=Auto`), working out the right form per connection: + +| | connected address is private | otherwise | +|---|---|---| +| **without TLS** | `internal-ip` | `external-ip` | +| **with TLS** | `internal-fqdn` | `external-fqdn` | + +The TLS split is about certificate validation: a certificate carrying DNS names cannot validate a bare address, +so an encrypted connection asks for names. Where there is no address to classify - a tunnel, a custom transport, +a Unix domain socket - the client asks for `none` rather than guessing, and falls back to reconnecting the way it +originally connected. + +This matters more than it sounds. Without a named replacement, a handoff has to wait for DNS to be repointed, +and DNS has been measured trailing the notification by anywhere from 4 to 19 seconds while the socket closes at +about 16 to 19 seconds - so on a bad run the connection is gone before DNS is ready. With a named replacement +the client moves within a second and the server never has to close anything. + +Override it if your deployment needs a specific form: + +``` +maintMovingEndpointType=ExternalFqdn +``` + +or `ServerDefault` to ask for nothing at all, which is what earlier versions did. + +## What the client does without your involvement + +| Notification | What the client does | +|---|---| +| `MIGRATING`, `FAILING_OVER`, `SMIGRATING` | relaxes command timeouts for that server while the disruption lasts | +| `MIGRATED`, `FAILED_OVER` | ends the window, but keeps timeouts relaxed for a short tail while things settle | +| `SMIGRATED` | as above, and re-reads the cluster topology, and re-subscribes any sharded channels whose slots moved | +| `MOVING` | works out the replacement address, lets in-flight work finish, then replaces the connections | + +So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. + +### Notifications that arrive as you connect + +Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_OVER` - and replays it to each connection that opts in, so a client that connects after a disruption still learns that it happened. Measured behaviour, worth knowing if you handle these events yourself: + +* Only completions are replayed. Starters (`MIGRATING`, `FAILING_OVER`) are not, and neither is `MOVING` - so a replay can never demand that you move. +* One item, most-recent-replaces; there is no queue. +* It arrives within milliseconds of the opt-in being accepted, which is *during* connection establishment - so an event handler attached after `ConnectAsync` returns will usually not see it. +* **It can be very old.** The same `FAILED_OVER` was still being replayed to fresh connections **three hours** after the failover - the longest anybody has measured, and it had not expired then - and a completion carries no time field, so nothing in the notification says how old it is. + +Because of that last point, a replayed completion is **not surfaced at all**: it does not relax timeouts, and `ServerMaintenanceEvent` is not raised for it. Nothing in the frame says whether the failover was seconds or hours ago, so any action taken on it would be a guess - and it is not ignored quietly, it is logged (see below), which is the right place for "here is what the server mentioned on the way in". + +That applies to the retained kinds only - `MIGRATED` and `FAILED_OVER`. A *starter* arriving as you connect is not a replay, and neither is `SMIGRATED`; nothing retains those, so one arriving mid-handshake is the server telling a late-joining connection about a disruption in progress. Those are raised and do relax timeouts, which is when patience is most useful. + +Note that a deliberate handoff appears as a `ConnectionFailed` event with `FailureType == ConnectionFailureType.MaintenanceHandoff`. That is expected during planned maintenance and does not indicate a fault; if you alert on `ConnectionFailed`, filter it out. + +## Watching the events + +```csharp +multiplexer.ServerMaintenanceEvent += (sender, e) => +{ + if (e is PushMaintenanceEvent maintenance) + { + logger.LogInformation( + "{Type} from {EndPoint} (seq {Sequence}, {Time})", + maintenance.NotificationType, maintenance.EndPoint, maintenance.SequenceId, maintenance.Time); + + foreach (var migration in maintenance.SlotMigrations) + { + logger.LogInformation("slots {Slots}: {Source} -> {Target}", migration.RawSlots, migration.Source, migration.Target); + } + } +}; +``` + +Two things are worth knowing before you build on the detail: + +* **`EndPoint` is whichever node told us first.** Every node broadcasts a given event, so the client collapses the copies and raises one event; it is not necessarily the node being maintained, and for the cluster notifications it is usually a bystander reporting somebody else's movements. +* **`SequenceId` identifies the event, not the delivery.** Every node that broadcasts a given event carries the same value, so the same notification arriving from several proxies - or replayed after a reconnect - is recognisable as one event. Ids are allocated per database and shared across notification types, so a `SMIGRATING` at 16 is followed by its `SMIGRATED` at 17. Use it for correlation and de-duplication rather than as an arithmetic sequence: gaps are normal, because a client only sees the events relevant to it. + +`Time` is what the server announced, and may legitimately be zero or negative for a connection that arrived mid-window, meaning "this is happening now". + +## Timeouts during maintenance + +Three settings control the relaxed window, all in seconds: + +| Setting | Default | Meaning | +|---|---|---| +| `maintRelaxedTimeout` | 10s | the timeout to use while a disruption is in progress, and the floor for how long a window lasts | +| `maintRelaxedWindowMax` | 3x the relaxed timeout | the longest a single window may last, in case a closing notification never arrives | +| `maintPostEventRelaxed` | 2x the relaxed timeout | how long timeouts stay relaxed *after* the disruption ends | + +The tail applies to a completion that arrives on a live connection. A completion replayed as you connect gets no tail at all, for the reasons above. + +The announced duration is clamped rather than honoured literally. Windows as short as two seconds have been observed in practice, which is not long enough to cover a client reconnecting, and a client that trusted the announced value would stop being patient exactly when it mattered. The tail exists for the same reason in reverse: after a handoff, servers and other clients are still settling. + +If a command does time out during a window, the exception carries the reason: `RedisTimeoutException.MaintenanceType` (and the same property on `RedisConnectionException`) names the notification that was in effect, which distinguishes "the deployment was moving" from "this query is slow". A window that closed very recently still counts, because timeouts are reported by a once-a-second sweep and the command had already been waiting for its whole timeout before that - so the window that caused a timeout is often over by the time you see the exception. + +A handoff that does not get a replacement connection fully established before the announced window runs out is reported as a warning: + +``` +10.0.0.1:6379: Maintenance handoff did not establish a replacement within the announced 15000ms (interactive: False, subscription: True) +``` + +Worth watching for, because it is otherwise invisible: commands succeed either way, since the relaxed window covers the gap, so a handoff that took three times its budget looks exactly like one that worked. + +## Checking that it is working + +Wire up an `ILoggerFactory` and the client reports the outcome of the opt-in, per server: + +```csharp +options.LoggerFactory = loggerFactory; +await using var multiplexer = await ConnectionMultiplexer.ConnectAsync(options); +``` + +``` +10.0.0.1:6379: Requesting maintenance notifications (Auto) +10.0.0.1:6379: Maintenance notifications accepted +``` + +or, when the server declines, the reason it gave: + +``` +10.0.0.1:6379: Maintenance notifications refused (ERR maintenance notifications are disabled on this server) +``` + +Received notifications are logged too, which is the quickest way to answer "did anything actually arrive?" - and, when a new connection is unexpectedly patient about timeouts, "was that a replay?": + +``` +10.0.0.1:6379: Maintenance notification: FailingOver seq=41 +10.0.0.1:6379: Maintenance notification: FailedOver seq=42 +10.0.0.2:6379: Maintenance notification: FailedOver seq=42 (catch-up) +``` + +The last line is the retained copy described above, delivered to a connection that opted in afterwards. + +A handoff is reported the same way, which is worth knowing because it replaces connections: + +``` +10.0.0.1:6379: Maintenance handoff: Recycle -> 10.0.0.2:6379: db.example.com now resolves to 10.0.0.2:6379 +``` + +Alternatively set `MaintenanceNotifications = Enabled` in a test or staging environment: if anything prevents the feature working, including ending up on RESP2, the connection fails instead of running silently without it. + +## Which deployments send these + +Redis Enterprise and Redis Cloud send them, subject to the feature being enabled on the cluster. Azure Managed Redis is configured to ask for them ahead of its own rollout, so the setting is harmless until their servers begin emitting. Redis Open Source, Valkey and other servers do not send them at all, and the setting is simply inert there: the opt-in is refused and the client carries on. + +# Azure Cache for Redis maintenance events (pub/sub) + +The original support, and unrelated to the push-frame mechanism above: Azure Cache for Redis publishes maintenance notifications on the `AzureRedisEvents` pub/sub channel, and the client reports them without changing its own behaviour. + ## Types of events Azure Cache for Redis currently sends the following notifications: @@ -64,4 +287,4 @@ It's important to understand that this does *not* mean downtime if you are using #### NodeMaintenanceEnded event -`NodeMaintenanceEnded` events are raised to indicate that the maintenance operation has completed and that the replica is once again available. You do *NOT* need to wait for this event to use the load balancer endpoint, as it is available throughout. However, we included this for logging purposes and for customers who use the replica endpoint in clusters for read workloads. \ No newline at end of file +`NodeMaintenanceEnded` events are raised to indicate that the maintenance operation has completed and that the replica is once again available. You do *NOT* need to wait for this event to use the load balancer endpoint, as it is available throughout. However, we included this for logging purposes and for customers who use the replica endpoint in clusters for read workloads. diff --git a/docs/exp/SER010.md b/docs/exp/SER010.md new file mode 100644 index 000000000..a09ccd279 --- /dev/null +++ b/docs/exp/SER010.md @@ -0,0 +1,58 @@ +Maintenance notifications ("smart client handoffs") are a server feature by which a server warns a +connected client in advance of a disruptive event - a shard migration, a failover, or the endpoint +itself being replaced - so the client can act before the disruption rather than react to a broken +connection afterwards. The client opts in per connection with `CLIENT MAINT_NOTIFICATIONS ON`, and the +notifications then arrive as RESP3 push frames on the connection that carries commands. + +The feature is experimental here for three separate reasons: + +1. **The server side is not universal.** Only Redis Enterprise and Redis Cloud emit these + notifications today; OSS Redis, Valkey and Garnet do not recognize the opt-in at all. That is why + `MaintenanceNotificationMode.Auto` exists, and why the default is `Disabled`. +2. **The wire contract is still evolving.** Notification types have been proposed and withdrawn during + its development, and the payloads are described in prose rather than pinned down by a formal + specification. Our parsing is validated against frames captured from live deployments and is + deliberately liberal about what it accepts, but the shapes may still change. +3. **What the client *does* in response is the substantial part**, and it is being built in stages - + timeout relaxation, then endpoint handoff. Behaviour may therefore change materially between + versions while the diagnostic is in place, even where the API does not. + +`MaintenanceNotificationMode`: + +- `Disabled` (the default) - never ask; the server sends nothing. +- `Enabled` - **required**: ask, and *reject the connection* unless notifications end up live. That covers a + server that refuses, a server that answers `HELLO 3` as RESP2, and a configuration that could never ask in + the first place (`Protocol = Resp2`, or `HELLO` unavailable) - requiring a RESP3-only feature over RESP2 is + a contradiction, and it fails rather than being half-honoured. Only point this at a deployment you know + supports them. +- `Auto` - ask, and carry on if the server refuses or the connection ends up RESP2; the feature is then + simply off for that server and the connection is never rejected over it. Safe against a mixture of + servers, or one whose support you don't know, and what most callers want. + +The names are the cross-client ones (they match go-redis and redis-py, so a connection string ports +between clients), which makes `Enabled` easy to misread as "on" - it means "required". One deliberate +divergence: the cross-client spec only calls for interrupting the connection when the *server* errors, +whereas we extend that to the RESP2 cases above, on the grounds that a required feature which cannot +possibly be delivered is the same failure either way. + +## A deployment prerequisite worth knowing + +On Redis Enterprise there are *two* switches, and they are easy to conflate. A **cluster-level flag** decides +whether the subcommand exists at all - `client_maint_notifications` for proxy-routed databases, +`oss_cluster_client_maint_notifications` for `oss_cluster` ones - and the **per-connection opt-in** this option +controls asks one connection to receive them. A cluster running a supporting version with the flag off will +refuse `CLIENT MAINT_NOTIFICATIONS ON`, which `Auto` absorbs silently and `Enabled` turns into a refused +connection. So if `Enabled` is rejecting connections against a deployment you believe supports the feature, +check the cluster flag before suspecting the client. + +If you accept the above, you can suppress this warning by adding the following to your `csproj` file: + +```xml +$(NoWarn);SER010 +``` + +or more granularly / locally in C#: + +``` c# +#pragma warning disable SER010 +``` diff --git a/docs/index.md b/docs/index.md index 93c6eab7d..aacd8021f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,7 +45,7 @@ Documentation - [Pub/Sub Key Notifications](KeyspaceNotifications) - how to use keyspace and keyevent notifications - [Hot Keys](HotKeys) - how to use `HOTKEYS` profiling - [Using RESP3](Resp3) - information on using RESP3 -- [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance (e.g. Azure Cache for Redis) +- [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance, including the server-native notifications sent by Redis Enterprise and Redis Cloud (known elsewhere as *smart client handoffs* or *hitless upgrades*) - [Streams](Streams) - how to use the Stream data type - [Arrays](Arrays) - how to use Redis Arrays as sparse arrays of values - [Vector Sets](VectorSets) - how to use Vector Sets for similarity search with embeddings diff --git a/src/RESPite/Shared/Experiments.cs b/src/RESPite/Shared/Experiments.cs index 2cfcd94d6..7077164ad 100644 --- a/src/RESPite/Shared/Experiments.cs +++ b/src/RESPite/Shared/Experiments.cs @@ -21,6 +21,7 @@ internal static class Experiments public const string GeoRedundantFailover = "SER007"; public const string Server_8_10 = "SER008"; public const string Transport = "SER009"; + public const string MaintenanceNotifications = "SER010"; // ReSharper restore InconsistentNaming diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index d533d4506..e8b800521 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -39,11 +39,13 @@ public FaultContext(Exception fault) kind = RedisErrorKind.ConnectionFault; flags = connection.Flags; status = connection.CommandStatus; + MaintenanceType = connection.MaintenanceType; break; case RedisTimeoutException timeout: kind = RedisErrorKind.Timeout; flags = timeout.Flags; status = timeout.Commandstatus; + MaintenanceType = timeout.MaintenanceType; break; case TimeoutException: kind = RedisErrorKind.Timeout; @@ -118,6 +120,18 @@ public FaultContext(Exception fault) /// public ConnectionFailureType ConnectionFailureType => _connectionFailureType; + /// + /// The maintenance notification in force when this fault happened, if any. + /// + /// + /// Useful to a circuit breaker: a fault during announced maintenance is expected and transient, and + /// counting it towards a trip that then withdraws a whole server is the opposite of what the notification + /// was for. Left to policy rather than acted on here, since "ignore faults during maintenance" is a + /// judgement about the deployment, not about the protocol. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public Maintenance.MaintenanceNotificationType MaintenanceType { get; } + private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) { // the client never handed it to the socket, so the server cannot have seen it diff --git a/src/StackExchange.Redis/Availability/HealthCheckContext.cs b/src/StackExchange.Redis/Availability/HealthCheckContext.cs index 6d3fd5037..3322ef421 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckContext.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -27,4 +27,16 @@ public readonly struct HealthCheckContext(IServer server, TimeSpan probeTimeout) /// themselves (the caller applies it), but may use it to bound any state they create. /// public TimeSpan ProbeTimeout => probeTimeout; + + /// + /// Gets flags that a probe should include on every command it issues. + /// + /// + /// This marks the traffic as a health check rather than as work a caller is waiting on. It matters because + /// idleness is what decides whether an endpoint can be given up: a probe that looks like caller work makes + /// a server appear busy precisely because we are watching it, and an endpoint that has left the deployment + /// then never gets retired. The built-in probes pass this; a custom probe should too, and passing it + /// changes nothing else about how the command is routed or queued. + /// + public CommandFlags ProbeFlags => Message.ProbeFlag; } diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs index 64b1e882d..3e1491a6c 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs @@ -1,4 +1,4 @@ -using System.Threading.Tasks; +using System.Threading.Tasks; namespace StackExchange.Redis.Availability; @@ -16,7 +16,7 @@ private PingProbe() { } public override async Task CheckHealthAsync(HealthCheckContext context) { - await context.Server.PingAsync(); + await context.Server.PingAsync(context.ProbeFlags); return HealthCheckResult.Healthy; } } diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs index 6e1524b7a..058cff727 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Buffers; using System.Threading.Tasks; @@ -39,10 +39,10 @@ await database.LockTakeAsync( key: key, value: payload, expiry: context.ProbeTimeout, - flags: CommandFlags.FireAndForget).ForAwait(); + flags: CommandFlags.FireAndForget | context.ProbeFlags).ForAwait(); // release from the db if matches (otherwise, we have no clue what happened, so: leave alone) - var success = await database.LockReleaseAsync(key, payload).ForAwait(); + var success = await database.LockReleaseAsync(key, payload, context.ProbeFlags).ForAwait(); return success ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; } finally diff --git a/src/StackExchange.Redis/ClusterConfiguration.cs b/src/StackExchange.Redis/ClusterConfiguration.cs index 2880cf6ac..08fcab15e 100644 --- a/src/StackExchange.Redis/ClusterConfiguration.cs +++ b/src/StackExchange.Redis/ClusterConfiguration.cs @@ -137,21 +137,93 @@ public override int GetHashCode() internal bool Includes(int hashSlot) => hashSlot >= from && hashSlot <= to; - private static bool TryParseInt16(string s, int offset, int count, out short value) + /// + /// Parses one range from within a larger string, so that a comma-separated list can be read without + /// allocating a substring per element. + /// + internal static bool TryParse(string value, int offset, int length, out SlotRange range) { - checked + range = default; + if (length <= 0) return false; + + int dash = -1; + for (int i = 0; i < length; i++) + { + if (value[offset + i] == '-') + { + dash = i; + break; + } + } + + if (dash < 0) { - value = 0; - int tmp = 0; - for (int i = 0; i < count; i++) + if (TryParseInt16(value, offset, length, out var only)) { - char c = s[offset + i]; - if (c < '0' || c > '9') return false; - tmp = (tmp * 10) + (c - '0'); + range = new SlotRange(only, only); + return true; } - value = (short)tmp; + return false; + } + + if (dash != 0 && dash != length - 1 + && TryParseInt16(value, offset, dash, out var from) + && TryParseInt16(value, offset + dash + 1, length - dash - 1, out var to)) + { + // a reversed range is a server bug, not something to normalize silently + if (from > to) return false; + range = new SlotRange(from, to); return true; } + + return false; + } + + /// + /// Parses the comma-and-range slot form used by the maintenance notifications, e.g. + /// 123,456,789-1000. Empty elements are skipped; a malformed element fails the whole list, + /// since a partially-read slot set is worse than none. + /// + internal static bool TryParseList(string? value, out List ranges) + { + ranges = []; + if (string.IsNullOrEmpty(value)) return false; + + int start = 0; + while (start <= value!.Length) + { + var comma = value.IndexOf(',', start); + var length = (comma < 0 ? value.Length : comma) - start; + if (length > 0) + { + if (!TryParse(value, start, length, out var range)) return false; + ranges.Add(range); + } + + if (comma < 0) break; + start = comma + 1; + } + + return ranges.Count != 0; + } + + private static bool TryParseInt16(string s, int offset, int count, out short value) + { + // note this deliberately does not use `checked`: it used to, and an out-of-range slot number + // therefore threw OverflowException from a Try* method - reachable from the public + // SlotRange.TryParse and from CLUSTER NODES parsing, and now also from the maintenance + // notification reader, where throwing on the read loop is far worse than rejecting a value + value = 0; + int tmp = 0; + for (int i = 0; i < count; i++) + { + char c = s[offset + i]; + if (c < '0' || c > '9') return false; + tmp = (tmp * 10) + (c - '0'); + if (tmp > short.MaxValue) return false; // as soon as it cannot fit, stop + } + value = (short)tmp; + return true; } int IComparable.CompareTo(object? obj) => obj is SlotRange sRange ? CompareTo(sRange) : -1; diff --git a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs index 42bb2fcbf..51e2caacf 100644 --- a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs @@ -1,6 +1,8 @@ using System; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Threading.Tasks; +using RESPite; namespace StackExchange.Redis.Configuration { @@ -29,6 +31,9 @@ public class AzureManagedRedisOptionsProvider : DefaultOptionsProvider ".redisenterprise.cache.azure.net", ]; + /// + public override string Name => "amr"; + /// public override bool IsMatch(EndPoint endpoint) { @@ -65,5 +70,18 @@ public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action public override string ConfigurationChannel => ""; // disable on AMR + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// Pre-emptive: AMR does not emit these yet, and support is being added concurrently with this + /// client-side work. is what makes that safe - until + /// the server side ships, the opt-in is refused and the feature stays off, and it then starts working + /// without anybody needing to change a connection string. AMR also already prefers RESP3 here, which + /// the feature requires. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; } } diff --git a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs index c02f8f760..ee577eb59 100644 --- a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs @@ -33,6 +33,9 @@ public class AzureOptionsProvider : DefaultOptionsProvider ".redis.cache.sovcloud-api.fr", }; + /// + public override string Name => "azure"; + /// public override bool IsMatch(EndPoint endpoint) { diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index afcce011f..055c3d23d 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using RESPite; namespace StackExchange.Redis.Configuration { @@ -27,6 +29,10 @@ public class DefaultOptionsProvider { new AzureOptionsProvider(), new AzureManagedRedisOptionsProvider(), + new RedisCloudOptionsProvider(), + + // matches nothing, so its position is irrelevant; it is here to be resolvable by name + new RedisEnterpriseOptionsProvider(), }; /// @@ -51,6 +57,68 @@ public static void AddProvider(DefaultOptionsProvider provider) /// public virtual bool IsMatch(EndPoint endpoint) => false; + /// + /// The name this provider can be selected by in a configuration string, as defaults={name}; + /// null (the default) means it cannot be named, and can only be selected in code or by + /// . + /// + /// + /// This exists for the deployments cannot recognize: an on-premise + /// Enterprise cluster has arbitrary DNS, and a hosted one reached through private DNS or a proxy no + /// longer looks like itself. Naming is also what makes a provider expressible in the connection string + /// an application is configured with, rather than only in code it would have to be rebuilt to change. + /// + /// Only providers registered with or built in can be + /// resolved by name - deliberately, since a configuration string that could name an arbitrary type + /// would be a way to have one loaded, and would defeat trimming. + /// + /// + public virtual string? Name => null; + + /// + /// The provider's where it has one, and the type name otherwise. + /// + /// + /// Display only - a provider in a log should read as amr rather than as a namespace-qualified + /// type name. Note that serialization tests directly rather than going through + /// here: this never returns null, and an unnameable provider must not end up in a + /// configuration string. + /// + public override string ToString() => Name ?? base.ToString()!; + + /// + /// Finds a registered provider by its . + /// + internal static bool TryGetByName(string name, [NotNullWhen(true)] out DefaultOptionsProvider? provider) + { + foreach (var candidate in KnownProviders) + { + if (candidate.Name is { } candidateName && string.Equals(candidateName, name, StringComparison.OrdinalIgnoreCase)) + { + provider = candidate; + return true; + } + } + + provider = null; + return false; + } + + /// + /// The names that defaults= accepts, for diagnostics. + /// + internal static string GetKnownNames() + { + var names = new List(); + foreach (var candidate in KnownProviders) + { + if (candidate.Name is { } name && !names.Contains(name)) names.Add(name); + } + + names.Sort(StringComparer.OrdinalIgnoreCase); + return string.Join(", ", names); + } + /// /// Gets a provider for the given endpoints, falling back to if nothing more specific is found. /// @@ -211,10 +279,25 @@ public static DefaultOptionsProvider GetProvider(EndPoint endpoint) public virtual string TieBreaker => "__Booksleeve_TieBreak"; /// - /// Check configuration every n interval. + /// Gets how often to re-check the replication role of each connected server, or + /// to never do so. /// + /// + /// An INFO replication on each established interactive connection, which also serves as the + /// keep-alive for those sockets. Not a topology re-read - see . + /// public virtual TimeSpan ConfigCheckInterval => TimeSpan.FromMinutes(1); + /// + /// Gets how often to re-read the deployment's topology when nothing has gone wrong, or + /// to never do so. + /// + /// + /// Long by design: this is a backstop for a topology change that produced no failure, no redirect and + /// no announcement, and its cost is paid by every client on the schedule at once. + /// + public virtual TimeSpan TopologyRefreshInterval => TimeSpan.FromMinutes(30); + /// /// The username to use to authenticate with the server. /// @@ -267,6 +350,79 @@ protected virtual string GetDefaultClientName() => /// public virtual RedisProtocol? Protocol => null; + /// + /// Gets whether to ask servers to send maintenance notifications. + /// + /// + /// here on purpose. The notification contract asks + /// clients to default to auto, and that is right for the deployments that emit them - but this + /// library is pointed at every RESP implementation there is, most of which have never heard of the + /// opt-in, and asking them all costs a handshake slot plus an error reply in their logs. Nor is the + /// server the only thing in the path: proxies sit in front of these deployments, and an unrecognized + /// CLIENT subcommand is not guaranteed to be answered as politely as a server would. A provider + /// that recognizes a deployment which supports the feature should override this. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Disabled; + + /// + /// Which form of address to ask a server to name when an endpoint moves. + /// + /// + /// : derive it per connection, and ask. A bare opt-in leaves + /// the choice to the server, and measurement showed what that means in practice - every MOVING + /// observed that way named no replacement at all, so a handoff had to wait for DNS, which trails the + /// notification by anywhere from four to nineteen seconds. Asking produces an address immediately. + /// + /// Safe to ask for: a server whose metadata lacks the parameter, or lacks the specific form requested, + /// answers with a null endpoint rather than an error - which is exactly the behaviour of not asking. So + /// the downside of the request is nothing, and the upside is a handoff that does not race DNS. If a + /// deployment is ever seen *refusing* the parameter outright, the fix is to remember that per server and + /// fall back to a bare opt-in; nothing observed so far needs it. + /// + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual MaintenanceEndpointType MaintenanceMovingEndpointType => MaintenanceEndpointType.Auto; + + /// + /// Gets the value command timeouts are relaxed to during an announced disruption; 10 seconds, as the + /// notification contract prescribes. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan MaintenanceRelaxedTimeout => TimeSpan.FromSeconds(10); + + /// + /// Gets the longest a relaxed window may last, or null to derive it as three times the + /// effective . + /// + /// + /// Deriving is the default because the two have to stay coherent: a cap below the timeout it bounds is + /// nonsense, and it is the *configured* relaxed timeout that matters, which this type cannot see. + /// At the prescribed 10 seconds that gives 30 - the contract caps the MOVING budget at 15, so + /// there is ample room for a legitimate window while a stuck one is bounded to half a minute. Return a + /// value here to pin it regardless of the relaxed timeout. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan? MaintenanceRelaxedWindowMax => null; + + /// + /// Gets how long timeouts stay relaxed after a disruption reports completion, or null to derive + /// it as twice the effective . + /// + /// + /// A completing notification says the server-side operation finished, not that the server is back to + /// normal latency - and the moment after one is precisely when every client that received the same + /// notification re-engages at once. That herd is the reason for a tail, and it scales with client + /// count, which is not something a client can observe. + /// + /// Note this is a deliberate divergence: go-redis and node-redis both treat a completion as + /// "stop being relaxed" and keep no tail at all (confirmed by their maintainers, 2026-09-02). Set this + /// to for that behaviour. + /// + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public virtual TimeSpan? MaintenancePostEventRelaxedDuration => null; + /// /// Gets whether to enable TCP keep-alive when appropriate (endpoint- and platform-dependent). /// diff --git a/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs b/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs new file mode 100644 index 000000000..fc00c4f9b --- /dev/null +++ b/src/StackExchange.Redis/Configuration/RedisCloudOptionsProvider.cs @@ -0,0 +1,83 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Configuration +{ + /// + /// Options provider for Redis Cloud environments. + /// + /// + /// Deliberately *not* a copy of , despite the deployments + /// looking similar: AMR is TLS-only and has a 7.4 floor, and Redis Cloud is neither. See the individual + /// members for what is shared and what is not. + /// + public class RedisCloudOptionsProvider : DefaultOptionsProvider + { + // note the third subsumes the first (EndsWith), so it is kept for the documentation rather than the + // logic - the narrower entry records what the common case actually looks like + private static readonly string[] redisCloudDomains = + [ + ".cloud.redislabs.com", // standard fully-managed endpoints (AWS, GCP, Azure BYOC) + ".cloud.redis.io", // newer routing scheme for managed instances + ".redislabs.com", // older subscriptions, addressed directly + ]; + + /// + public override string Name => "rediscloud"; + + /// + public override bool IsMatch(EndPoint endpoint) + => endpoint is DnsEndPoint dnsEp && IsHostInDomains(dnsEp.Host, redisCloudDomains); + + private static bool IsHostInDomains(string hostName, string[] domains) + { + foreach (var domain in domains) + { + if (hostName.EndsWith(domain, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } + + /// + /// Allow connecting after startup, in the cases where the remote cache isn't ready or is overloaded. + /// + public override bool AbortOnConnectFail => false; + + /// + /// Prefer RESP3, which the deployment supports and which maintenance notifications require. + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; + + /// + /// Disabled: the deployment is proxied, so the OSS configuration-broadcast channel conveys nothing. + /// + public override string ConfigurationChannel => ""; + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// This is the deployment family the feature exists for. + /// rather than because a database that has not been + /// updated yet must keep working: the opt-in is then refused and the feature stays off, rather than the + /// connection being rejected. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; + + // Two things AzureManagedRedisOptionsProvider does that are deliberately *not* repeated here: + // + // - GetDefaultSsl => true. AMR is TLS-only, so assuming TLS there is safe. Redis Cloud enables TLS + // per database and plenty of databases are plaintext, so defaulting it on would fail their connect + // outright - a much worse outcome than not having guessed. + // - DefaultVersion => 7.4. That is AMR's floor. Redis Cloud still offers older versions per database, + // and claiming a version we do not have unlocks commands the server will reject, so the base + // default stands. + } +} diff --git a/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs b/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs new file mode 100644 index 000000000..87634fbf5 --- /dev/null +++ b/src/StackExchange.Redis/Configuration/RedisEnterpriseOptionsProvider.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Configuration +{ + /// + /// Options provider for a self-managed Redis Enterprise deployment, selected explicitly. + /// + /// + /// Deliberately matches no endpoint. An on-premise cluster has whatever DNS its operator gave it, so there + /// is nothing to recognize - which is exactly why this provider is nameable: it can be asked for as + /// defaults=enterprise in a configuration string, or assigned to + /// in code. + /// + /// It is also the right choice for a hosted deployment reached somewhere its own provider cannot see it - + /// behind private DNS, a CNAME, or a proxy - where the endpoint no longer looks like what it is. + /// + /// + public class RedisEnterpriseOptionsProvider : DefaultOptionsProvider + { + /// + public override string Name => "enterprise"; + + /// + /// Prefer RESP3, which the deployment supports and which maintenance notifications require. + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; + + /// + /// Disabled: the deployment is proxied, so the OSS configuration-broadcast channel conveys nothing. + /// + public override string ConfigurationChannel => ""; + + /// + /// Ask for maintenance notifications, tolerating a server that doesn't offer them. + /// + /// + /// rather than + /// : a cluster that has not been updated yet, or has + /// the feature switched off, must keep working. Choose Enabled explicitly if you would rather a + /// connection be refused than run without advance warning. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public override MaintenanceNotificationMode MaintenanceNotifications => MaintenanceNotificationMode.Auto; + + // Note: no GetDefaultSsl and no DefaultVersion override. Both are deployment choices here rather than + // properties of the product - TLS is configured per database, and the version is whatever was + // installed - so guessing either would be worse than leaving the library's defaults in place. + } +} diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 12db0bac1..3fd822ea3 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -73,6 +73,58 @@ internal static Proxy ParseProxy(string key, string value) return tmp; } + internal static DefaultOptionsProvider ParseDefaultsProvider(string key, string value) + { + if (!DefaultOptionsProvider.TryGetByName(value, out var provider)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a known defaults provider name; '{value}' is not one of: {DefaultOptionsProvider.GetKnownNames()}."); + } + return provider; + } + + internal static MaintenanceNotificationMode ParseMaintenanceNotifications(string key, string value) + { + if (!Enum.TryParse(value, true, out MaintenanceNotificationMode tmp) || !Enum.IsDefined(typeof(MaintenanceNotificationMode), tmp)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a MaintenanceNotificationMode value; the value '{value}' is not recognised."); + } + return tmp; + } + + internal static MaintenanceEndpointType ParseMaintenanceEndpointType(string key, string value) + { + if (!Enum.TryParse(value, true, out MaintenanceEndpointType tmp) || !Enum.IsDefined(typeof(MaintenanceEndpointType), tmp)) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a MaintenanceEndpointType value; the value '{value}' is not recognised."); + } + return tmp; + } + + /// + /// Parses one of the maintenance durations, which are expressed in seconds - the unit the + /// cross-client contract uses for maintRelaxedTimeout, so a documented value can be pasted + /// between clients. + /// + /// + /// Seconds is the odd one out in this file, where every other timeout is milliseconds, and the + /// mistake is silent in one direction: a caller who assumes milliseconds and writes 30000 would + /// otherwise get an eight-hour relaxed timeout. Hence the upper bound, whose message names the + /// unit - it exists to turn that into a diagnosable error rather than a mystery. + /// + internal static TimeSpan ParseMaintenanceSeconds(string key, string value) + { + const int MaxSeconds = 600; + if (!Format.TryParseInt32(value, out int seconds) || seconds < 0) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a non-negative integer number of seconds; the value '{value}' is not valid."); + } + if (seconds > MaxSeconds) + { + throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' is expressed in seconds, and {seconds}s exceeds the maximum of {MaxSeconds}s; note that this option is not in milliseconds."); + } + return TimeSpan.FromSeconds(seconds); + } + internal static SslProtocols ParseSslProtocols(string key, string? value) { // Flags expect commas as separators, but we need to use '|' since commas are already used in the connection string to mean something else @@ -99,6 +151,7 @@ internal const string ChannelPrefix = "channelPrefix", ConfigChannel = "configChannel", ConfigCheckSeconds = "configCheckSeconds", + TopologyRefreshSeconds = "topologyRefreshSeconds", ConnectRetry = "connectRetry", ConnectTimeout = "connectTimeout", DefaultDatabase = "defaultDatabase", @@ -125,6 +178,12 @@ internal const string Tunnel = "tunnel", SetClientLibrary = "setlib", Protocol = "protocol", + Defaults = "defaults", + MaintenanceNotifications = "maintNotifications", + MaintenanceMovingEndpointType = "maintMovingEndpointType", + MaintenanceRelaxedTimeout = "maintRelaxedTimeout", + MaintenanceRelaxedWindowMax = "maintRelaxedWindowMax", + MaintenancePostEventRelaxedDuration = "maintPostEventRelaxed", HighIntegrity = "highIntegrity", TcpKeepAlive = "tcpKeepAlive"; @@ -137,6 +196,7 @@ internal const string ClientName, ConfigChannel, ConfigCheckSeconds, + TopologyRefreshSeconds, ConnectRetry, ConnectTimeout, DefaultDatabase, @@ -162,6 +222,12 @@ internal const string Tunnel, SetClientLibrary, Protocol, + Defaults, + MaintenanceMovingEndpointType, + MaintenanceNotifications, + MaintenanceRelaxedTimeout, + MaintenanceRelaxedWindowMax, + MaintenancePostEventRelaxedDuration, HighIntegrity, TcpKeepAlive, }.ToDictionary(x => x, StringComparer.OrdinalIgnoreCase); @@ -217,6 +283,13 @@ private enum OptionFlags : ulong SslProtocolsHasValue = 1UL << 32, ProtocolHasValue = 1UL << 33, AllowSimulateConnectionFailure = 1UL << 34, + MaintenanceNotificationsHasValue = 1UL << 35, + MaintenanceMovingEndpointTypeHasValue = 1UL << 40, + DefaultsHasValue = 1UL << 36, + MaintenanceRelaxedTimeoutHasValue = 1UL << 37, + MaintenanceRelaxedWindowMaxHasValue = 1UL << 38, + MaintenancePostEventRelaxedDurationHasValue = 1UL << 39, + TopologyRefreshSecondsHasValue = 1UL << 41, } private OptionFlags optionFlags; @@ -230,6 +303,7 @@ private enum OptionFlags : ulong private Version? defaultVersion; private int keepAlive, asyncTimeout, syncTimeout, connectTimeout, responseTimeout, connectRetry, configCheckSeconds, defaultDatabase; + private int topologyRefreshSeconds; private Proxy proxy; @@ -242,6 +316,9 @@ private enum OptionFlags : ulong private SslProtocols sslProtocols; private RedisProtocol _protocol; + private MaintenanceNotificationMode _maintenanceNotifications; + private MaintenanceEndpointType _maintenanceMovingEndpointType; + private TimeSpan _maintenanceRelaxedTimeout, _maintenanceRelaxedWindowMax, _maintenancePostEventRelaxedDuration; private bool HasValue(OptionFlags hasValue) => (optionFlags & hasValue) != 0; @@ -314,7 +391,15 @@ private void SetWithValue(OptionFlags hasValue, ref T storage, T value) where public DefaultOptionsProvider Defaults { get => defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); - set => defaultOptions = value; + set + { + defaultOptions = value; + + // the getter memoizes an *inferred* provider into the same field, so a flag is the only way to + // tell "the caller chose this" from "we worked it out from the endpoints" - and only the former + // may be written back out to a configuration string + optionFlags |= OptionFlags.DefaultsHasValue; + } } /// @@ -933,14 +1018,53 @@ internal RemoteCertificateValidationCallback? CertificateValidationCallback } /// - /// Check configuration every n seconds (every minute by default). + /// How often to re-check the replication role of each connected server, in seconds (every minute by + /// default), or 0 to never do so. /// + /// + /// Sends an INFO replication on each *established* interactive connection. That is how a + /// primary/replica change is noticed on a deployment that does not announce one, and it doubles as the + /// keep-alive for those sockets, which is why the interval is short. + /// + /// Despite the name, this is not a topology re-read. It asks a server we are already talking to what it + /// says about itself, so it reveals nothing about servers we cannot reach, about endpoints that have + /// left the deployment, or about cluster slot ownership. is the + /// setting for that, and is deliberately much less frequent because it costs much more. + /// + /// public int ConfigCheckSeconds { get => HasValue(OptionFlags.ConfigCheckSecondsHasValue) ? configCheckSeconds : (int)Defaults.ConfigCheckInterval.TotalSeconds; set => SetWithValue(OptionFlags.ConfigCheckSecondsHasValue, ref configCheckSeconds, value); } + /// + /// Re-read the deployment's topology every n seconds even when nothing has gone wrong, or 0 to + /// never do so (every 30 minutes by default). + /// + /// + /// A backstop, not the primary mechanism. Topology is normally learned from something happening: a + /// redirect from a reachable node, a configuration announcement, a maintenance notification, or a + /// connection failing. Each of those needs *somebody* to notice, and the case none of them covers is an + /// endpoint that is reachable, answers a handshake, and is no longer part of the deployment - no + /// failure, no redirect, nothing announced. + /// + /// The default is deliberately long, and each refresh is spread by up to 30 seconds of jitter, because + /// the cost of this is paid per client: a fleet of them re-reading configuration on the same schedule + /// is exactly the stampede that the failure-driven paths are careful to avoid. If you want it off, set + /// it to zero. + /// + /// + /// Distinct from , which despite its name does not re-read topology - + /// it sends an INFO replication on an established connection to check a server's role. + /// + /// + public int TopologyRefreshSeconds + { + get => HasValue(OptionFlags.TopologyRefreshSecondsHasValue) ? topologyRefreshSeconds : (int)Defaults.TopologyRefreshInterval.TotalSeconds; + set => SetWithValue(OptionFlags.TopologyRefreshSecondsHasValue, ref topologyRefreshSeconds, value); + } + /// /// Parse the configuration from a comma-delimited configuration string. /// @@ -990,6 +1114,7 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow #pragma warning restore CS0618 // Type or member is obsolete connectRetry = connectRetry, configCheckSeconds = configCheckSeconds, + topologyRefreshSeconds = topologyRefreshSeconds, responseTimeout = responseTimeout, defaultDatabase = defaultDatabase, reconnectRetryPolicy = reconnectRetryPolicy, @@ -1004,6 +1129,10 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow Tunnel = Tunnel, LibraryName = LibraryName, _protocol = _protocol, + _maintenanceNotifications = _maintenanceNotifications, + _maintenanceRelaxedTimeout = _maintenanceRelaxedTimeout, + _maintenanceRelaxedWindowMax = _maintenanceRelaxedWindowMax, + _maintenancePostEventRelaxedDuration = _maintenancePostEventRelaxedDuration, heartbeatInterval = heartbeatInterval, WriteMode = WriteMode, CircuitBreaker = CircuitBreaker, @@ -1092,11 +1221,20 @@ public string ToString(bool includePassword) Append(sb, OptionKeys.ConnectRetry, OptionFlags.ConnectRetryHasValue, in connectRetry); Append(sb, OptionKeys.Proxy, OptionFlags.ProxyHasValue, in proxy); Append(sb, OptionKeys.ConfigCheckSeconds, OptionFlags.ConfigCheckSecondsHasValue, in configCheckSeconds); + Append(sb, OptionKeys.TopologyRefreshSeconds, OptionFlags.TopologyRefreshSecondsHasValue, in topologyRefreshSeconds); Append(sb, OptionKeys.ResponseTimeout, OptionFlags.ResponseTimeoutHasValue, in responseTimeout); Append(sb, OptionKeys.DefaultDatabase, OptionFlags.DefaultDatabaseHasValue, in defaultDatabase); Append(sb, OptionKeys.SetClientLibrary, OptionFlags.SetClientLibraryHasValue, OptionFlags.SetClientLibraryValue); Append(sb, OptionKeys.HighIntegrity, OptionFlags.HighIntegrityHasValue, OptionFlags.HighIntegrityValue); if (HasValue(OptionFlags.ProtocolHasValue)) Append(sb, OptionKeys.Protocol, FormatProtocol(_protocol)); + // only when the caller set it *and* it can be named: an inferred provider must not be baked into + // the string, or re-parsing would pin a choice that was only ever a guess from the endpoints + if (HasValue(OptionFlags.DefaultsHasValue) && defaultOptions?.Name is { } defaultsName) Append(sb, OptionKeys.Defaults, defaultsName); + if (HasValue(OptionFlags.MaintenanceNotificationsHasValue)) Append(sb, OptionKeys.MaintenanceNotifications, _maintenanceNotifications.ToString()); + if (HasValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue)) Append(sb, OptionKeys.MaintenanceMovingEndpointType, _maintenanceMovingEndpointType.ToString()); + if (HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedTimeout, FormatMaintenanceSeconds(_maintenanceRelaxedTimeout)); + if (HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue)) Append(sb, OptionKeys.MaintenanceRelaxedWindowMax, FormatMaintenanceSeconds(_maintenanceRelaxedWindowMax)); + if (HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue)) Append(sb, OptionKeys.MaintenancePostEventRelaxedDuration, FormatMaintenanceSeconds(_maintenancePostEventRelaxedDuration)); Append(sb, OptionKeys.TcpKeepAlive, OptionFlags.TcpKeepAliveHasValue, OptionFlags.TcpKeepAliveValue); if (Tunnel is { IsInbuilt: true } tunnel) { @@ -1212,6 +1350,8 @@ private void Clear() #endif Tunnel = null; _protocol = default; + _maintenanceNotifications = default; + _maintenanceRelaxedTimeout = _maintenanceRelaxedWindowMax = _maintenancePostEventRelaxedDuration = default; WriteMode = default; CircuitBreaker = null; RetryPolicy = null; @@ -1295,6 +1435,9 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) case OptionKeys.ConnectRetry: ConnectRetry = OptionKeys.ParseInt32(key, value); break; + case OptionKeys.TopologyRefreshSeconds: + TopologyRefreshSeconds = OptionKeys.ParseInt32(key, value); + break; case OptionKeys.ConfigCheckSeconds: ConfigCheckSeconds = OptionKeys.ParseInt32(key, value); break; @@ -1368,6 +1511,24 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) case OptionKeys.Protocol: SetWithValue(OptionFlags.ProtocolHasValue, ref _protocol, OptionKeys.ParseRedisProtocol(key, value)); break; + case OptionKeys.Defaults: + Defaults = OptionKeys.ParseDefaultsProvider(key, value); + break; + case OptionKeys.MaintenanceNotifications: + SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, OptionKeys.ParseMaintenanceNotifications(key, value)); + break; + case OptionKeys.MaintenanceMovingEndpointType: + SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, OptionKeys.ParseMaintenanceEndpointType(key, value)); + break; + case OptionKeys.MaintenanceRelaxedTimeout: + SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; + case OptionKeys.MaintenanceRelaxedWindowMax: + SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; + case OptionKeys.MaintenancePostEventRelaxedDuration: + SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, OptionKeys.ParseMaintenanceSeconds(key, value)); + break; // Deprecated options we ignore... case OptionKeys.HighPrioritySocketThreads: case OptionKeys.PreserveAsyncOrder: @@ -1420,6 +1581,105 @@ public RedisProtocol? Protocol set => Set(OptionFlags.ProtocolHasValue, ref _protocol, value); } + /// + /// Whether to ask servers to send maintenance notifications; note that + /// means required and rejects connections + /// that cannot deliver them - is the best-effort + /// mode. + /// + /// + /// Requires RESP3, and only Redis Enterprise and Redis Cloud emit them - so the default is + /// rather than spending an extra handshake command + /// asking every server in existence a question almost none of them understand. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationMode MaintenanceNotifications + { + get => HasValue(OptionFlags.MaintenanceNotificationsHasValue) ? _maintenanceNotifications : Defaults.MaintenanceNotifications; + set => SetWithValue(OptionFlags.MaintenanceNotificationsHasValue, ref _maintenanceNotifications, value); + } + + /// + /// Which form of address a server should name when it announces that an endpoint is moving. + /// + /// + /// Sent as moving-endpoint-type on the maintenance-notification opt-in. + /// (the default) sends no preference at all, which + /// is what the client has always done - and every MOVING observed that way carried no address, so + /// a client that wants a named replacement should ask for one. Prefer an FQDN form under TLS: an address + /// cannot be validated against a certificate carrying only DNS names. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceEndpointType MaintenanceMovingEndpointType + { + get => HasValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue) + ? _maintenanceMovingEndpointType + : Defaults.MaintenanceMovingEndpointType; + set => SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, value); + } + + /// + /// The value command timeouts are relaxed to while a server has announced a disruption. + /// + /// + /// This is a floor, never a reduction: the effective timeout inside a window is + /// max(configured, this), so a caller with a generous timeout keeps it. Expressed in seconds in + /// a configuration string (maintRelaxedTimeout=10), matching the cross-client contract - note + /// that this is unlike every other timeout here, which are milliseconds. Only command timeouts are + /// relaxed; keep-alive, the heartbeat and connection-failure detection are deliberately untouched, so + /// a server that dies mid-maintenance is still noticed on the usual schedule. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenanceRelaxedTimeout + { + get => HasValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue) ? _maintenanceRelaxedTimeout : Defaults.MaintenanceRelaxedTimeout; + set => SetWithValue(OptionFlags.MaintenanceRelaxedTimeoutHasValue, ref _maintenanceRelaxedTimeout, value); + } + + /// + /// The longest a relaxed window may last, however long the server said the disruption would take. + /// + /// + /// A backstop for a closing notification that never arrives, and our invention - the + /// notification contract names no upper bound. A window that never closes is worse than one that + /// closes early, since relaxation delays the point at which a genuinely slow server surfaces as a + /// timeout. Expressed in seconds in a configuration string. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenanceRelaxedWindowMax + { + get => HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue) + ? _maintenanceRelaxedWindowMax + : Defaults.MaintenanceRelaxedWindowMax ?? Multiply(MaintenanceRelaxedTimeout, 3); + set => SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + } + + /// + /// How long to keep timeouts relaxed after a disruption reports that it has finished. + /// + /// + /// Also our invention. A closing notification means the server-side operation completed, not + /// that the server is back to normal latency - and the moment after it completes is precisely when + /// every other client that received the same notification re-engages, so the load spike arrives + /// slightly after the all-clear. Does not apply when a window ended by hitting + /// : in that case nothing told us the event finished, and + /// extending past the backstop would defeat it. Expressed in seconds in a configuration string. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public TimeSpan MaintenancePostEventRelaxedDuration + { + get => HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue) + ? _maintenancePostEventRelaxedDuration + : Defaults.MaintenancePostEventRelaxedDuration ?? Multiply(MaintenanceRelaxedTimeout, 2); + set => SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + } + + // TimeSpan * int is netstandard2.1+, so this is ticks arithmetic to keep the down-level TFMs building + private static TimeSpan Multiply(TimeSpan value, int factor) => TimeSpan.FromTicks(value.Ticks * factor); + + private static string FormatMaintenanceSeconds(TimeSpan value) + => ((int)value.TotalSeconds).ToString(System.Globalization.CultureInfo.InvariantCulture); + internal BufferedStreamWriter.WriteMode WriteMode { get; set; } /// diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs index 0a8b95be5..fd5bd7c7f 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Events.cs @@ -88,6 +88,69 @@ private void OnEndpointChanged(EndPoint endpoint, EventHandler public event EventHandler? ServerMaintenanceEvent; + + /// + /// How host names are resolved during a maintenance handoff. + /// + /// + /// A seam rather than a call to directly, because no in-process fake can move a + /// DNS record: the handoff's whole decision turns on the answer *changing*, and the only way to test that + /// deterministically is to supply the answers. Defaults to real DNS. + /// + internal Func> AddressResolver { get; set; } + = Maintenance.AdvertisedAddressProbe.DefaultResolveAsync; + + // recently-raised (type, sequence) pairs, so one logical event raises one event however many nodes told + // us. Small and fixed: the copies arrive within milliseconds of each other, so a handful of slots covers + // any realistic proxy count even with other notifications interleaved. + // A named struct rather than a tuple: this assembly must not reference System.ValueTuple, which breaks + // .NET Framework consumers - see SanityChecks.ValueTupleNotReferenced. + private readonly struct RaisedMaintenanceEvent(Maintenance.MaintenanceNotificationType type, long sequence) + { + public readonly Maintenance.MaintenanceNotificationType Type = type; + public readonly long Sequence = sequence; + } + + private readonly RaisedMaintenanceEvent[] _raisedMaintenanceEvents = new RaisedMaintenanceEvent[8]; + private int _raisedMaintenanceEventIndex; + + /// + /// Whether this is the first time we have been told about a given maintenance event, across every + /// connection. + /// + /// + /// Every node broadcasts a given event, and all of them carry the same sequence number - observed on + /// Enterprise 8.6.2, where the id identifies the event rather than the delivery. So without this, a + /// deployment fronted by three proxies raises three events for one migration and every consumer has to + /// dedupe them. + /// + /// Matched on equality rather than "less than or equal", deliberately: a lagging node reporting an + /// *earlier* event we have not seen yet is a distinct event and must still be raised. Only an exact repeat + /// of something already raised is a duplicate. + /// + /// + /// Eviction is the only expiry - an entry falls out once eight further notifications have been recorded, so + /// nothing has to be purged on a timer and the state cannot grow. A duplicate arriving after its entry has + /// been evicted would be raised a second time, which is the right way round to be wrong: the copies arrive + /// within milliseconds of each other, so that takes a straggler behind eight intervening events. + /// + /// + internal bool TryClaimMaintenanceEvent(Maintenance.MaintenanceNotificationType type, long? sequence) + { + if (sequence is not { } seq) return true; // no id to match on; better a duplicate than a silence + + lock (_raisedMaintenanceEvents) + { + foreach (var entry in _raisedMaintenanceEvents) + { + if (entry.Type == type && entry.Sequence == seq) return false; + } + + _raisedMaintenanceEvents[_raisedMaintenanceEventIndex] = new RaisedMaintenanceEvent(type, seq); + _raisedMaintenanceEventIndex = (_raisedMaintenanceEventIndex + 1) % _raisedMaintenanceEvents.Length; + return true; + } + } internal void OnServerMaintenanceEvent(ServerMaintenanceEvent e) => ServerMaintenanceEvent?.Invoke(this, e); diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 903941c1f..3a5261298 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1332,6 +1332,57 @@ public void UnRoot(int token) } } + private int _nextTopologyRefreshTicks; // 0 until the first heartbeat schedules one + + /// How far apart two clients' refreshes are spread; not configurable, because nobody needs to tune it. + private const int TopologyRefreshJitterMilliseconds = 30_000; + + /// + /// Re-reads the topology on a long timer, as a backstop for a change that nothing reported. + /// + /// + /// Every other refresh path is event-driven: a redirect, an announcement, a notification, or a + /// connection failing. They cover almost everything between them - the case left over is an endpoint + /// that is reachable, completes a handshake, and is no longer part of the deployment, which produces + /// none of those signals and so was previously invisible for the lifetime of the multiplexer. + /// + /// Two things keep the cost honest. The interval is long (30 minutes by default), and each client + /// picks its own phase within a 30-second jitter on every cycle, so a fleet started together does not + /// stay in step. Beyond that this is the ordinary refresh path, which already declines while another + /// is in flight. + /// + /// + /// The first interval is measured from the first heartbeat rather than from construction, so nothing + /// is read on behalf of a multiplexer that is created, used briefly and disposed. + /// + /// + private void CheckTopologyRefreshDue(int now) + { + var seconds = RawConfig.TopologyRefreshSeconds; + if (seconds <= 0 || _isDisposed) return; + + var next = Volatile.Read(ref _nextTopologyRefreshTicks); + if (next == 0) + { + Interlocked.CompareExchange(ref _nextTopologyRefreshTicks, ScheduleTopologyRefresh(now, seconds), 0); + return; + } + + if (unchecked(now - next) < 0) return; // not due yet + + // reschedule *before* refreshing, and only if nobody else got there first: a refresh that takes + // longer than a heartbeat must not queue a second one behind it + if (Interlocked.CompareExchange(ref _nextTopologyRefreshTicks, ScheduleTopologyRefresh(now, seconds), next) != next) return; + + ReconfigureIfNeeded(null, fromBroadcast: false, "periodic topology refresh"); + } + + private static int ScheduleTopologyRefresh(int now, int seconds) + { + var due = unchecked(now + (seconds * 1000) + ServerSelectionStrategy.SharedRandom.Next(TopologyRefreshJitterMilliseconds)); + return due == 0 ? 1 : due; // zero means "not scheduled", so never land on it + } + internal void OnHeartbeat() { try @@ -1341,6 +1392,8 @@ internal void OnHeartbeat() Interlocked.Exchange(ref lastGlobalHeartbeatTicks, now); Trace("heartbeat"); + CheckTopologyRefreshDue(now); + var tmp = GetServerSnapshot(); int token = 0; bool isRooted = pulse?.IsRooted(out token) ?? false, hasPendingCallerFacingItems = false; @@ -2492,7 +2545,25 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u throw GetException(result, message, server); } - if (Monitor.Wait(source, TimeoutMilliseconds)) + // Unlike the async sweeps, which re-read the effective timeout on every heartbeat, a + // sync caller commits to a duration when it parks - so if maintenance relaxation + // begins while we are waiting, we have to notice by re-waiting rather than failing at + // the original deadline. Without this a sync caller in flight when a MIGRATING + // arrives times out at the strict timeout while its async neighbour is relaxed. + var watch = ValueStopwatch.StartNew(); + bool completed = Monitor.Wait(source, server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds); + while (!completed) + { + var elapsed = watch.ElapsedMilliseconds; + var revised = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds; + + // also the path back: if a window closed while we waited, revised drops to the + // configured value and we stop + if (revised <= elapsed) break; + completed = Monitor.Wait(source, revised - elapsed); + } + + if (completed) { Trace("Timely response to " + message); } diff --git a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs index 9f0df5ba0..3fd7847b3 100644 --- a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs +++ b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs @@ -68,5 +68,22 @@ public enum ConnectionFailureType /// [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] CircuitBreaker, + + /// + /// The connection was replaced deliberately, to move off an endpoint the server announced it is + /// retiring. + /// + /// + /// Not a fault: the connection was working, and we chose to replace it while a replacement address was + /// known rather than wait to be disconnected. Reported here because the alternative is worse - the + /// replacement raises , so without this a consumer + /// tracking connection state sees a restore with no matching failure, and no reason for the churn. + /// + /// Consumers alerting on should filter this out: + /// it is expected during planned maintenance and says nothing is wrong. is + /// the precedent for a deliberate client action being reported this way. + /// + /// + MaintenanceHandoff, } } diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index aa9bf7001..bfd92ff5e 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -319,14 +319,29 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; + // If the server had announced a disruption, say so on the fault: "timeout" and "timeout during an + // announced failover" call for very different reactions from whoever reads the log. + // + // Deliberately not the *active* type. A command that timed out was outstanding for its whole + // timeout before the heartbeat noticed, so a window covering its entire life can already have + // closed by the time this runs - which used to report None for a timeout maintenance plainly + // caused. The timeout that applied is the bound on how far back to look; take the async one when + // this message was awaited, since the two can be configured very differently. + var applicableTimeout = message?.ResultBoxIsAsync == true + ? multiplexer.AsyncTimeoutMilliseconds + : multiplexer.TimeoutMilliseconds; + var maintenanceType = server?.GetMaintenanceTypeForFault(applicableTimeout) + ?? Maintenance.MaintenanceNotificationType.None; Exception ex = logConnectionException && lastConnectionException is not null ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, + MaintenanceType = maintenanceType, } : new RedisTimeoutException(msgFlags, sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, + MaintenanceType = maintenanceType, }; CopyDataToException(data, ex); diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 03c62ed95..e6a94edda 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using StackExchange.Redis.Maintenance; namespace StackExchange.Redis; @@ -765,4 +766,46 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) EventId = 110, Message = "{BridgeName}: Transport connected (encrypted: {IsEncrypted})")] internal static partial void LogInformationTransportConnected(this ILogger logger, string bridgeName, bool isEncrypted); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 116, + Message = "{Server}: Requesting maintenance notifications ({Mode})")] + internal static partial void LogInformationRequestingMaintenanceNotifications(this ILogger logger, ServerEndPointLogValue server, MaintenanceNotificationMode mode); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 117, + Message = "{Server}: Maintenance notifications accepted")] + internal static partial void LogInformationMaintenanceNotificationsAccepted(this ILogger logger, ServerEndPointLogValue server); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 118, + Message = "{Server}: Maintenance notifications refused ({Reason})")] + internal static partial void LogInformationMaintenanceNotificationsRefused(this ILogger logger, ServerEndPointLogValue server, string reason); + + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 122, + Message = "{Server}: Maintenance handoff did not establish a replacement within the announced {WindowMilliseconds}ms (interactive: {Interactive}, subscription: {Subscription})")] + internal static partial void LogWarningMaintenanceHandoffMissedDeadline(this ILogger logger, ServerEndPointLogValue server, long windowMilliseconds, bool interactive, bool subscription); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 121, + Message = "{Server}: Maintenance notification: {Type} seq={Sequence}{CatchUp}")] + internal static partial void LogInformationMaintenanceNotificationReceived(this ILogger logger, ServerEndPointLogValue server, MaintenanceNotificationType type, long sequence, string catchUp); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 119, + Message = "{Server}: Maintenance handoff: {Outcome}")] + internal static partial void LogInformationMaintenanceHandoff(this ILogger logger, ServerEndPointLogValue server, string outcome); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 120, + Message = "{Server}: Re-reading topology after {Failures} consecutive connect failures")] + internal static partial void LogInformationRefreshingAfterConnectFailures(this ILogger logger, ServerEndPointLogValue server, int failures); } diff --git a/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs b/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs new file mode 100644 index 000000000..6170e4dc1 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/AdvertisedAddressProbe.cs @@ -0,0 +1,227 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Maintenance; + +/// +/// Asks DNS the two questions a handoff needs: what replaces the address we are leaving, and are we still +/// advertised at all. +/// +/// +/// A poll rather than a lookup, and that is the whole point. Measured on Redis Enterprise (2026-08-28), with +/// times relative to the notification: the server-side endpoint moves at +8.6s, DNS follows at +9.7s and +4.4s +/// across two runs, and the sockets close at +18.4s and +16.6s - against a declared 15s grace and a 5s record +/// TTL. So resolving immediately returns the address being retired, in every run observed, and a +/// client that treats the first answer as authoritative hands off to the node it was told to leave. +/// +/// DNS is not guaranteed to win the race. On a second cluster the same notification with the same 15s +/// grace saw the socket close at +15.7s and DNS update at +18.7s - three seconds after the close. So +/// there are three outcomes here, not two, and the third is a normal one: the record may still be stale when +/// the window runs out, and the only thing left is to reconnect after the close and resolve then. Do not +/// "simplify" the null return away. +/// +/// +/// There is no way to observe the intermediate state from outside: the endpoint has moved server-side well +/// before DNS reflects it, and nothing tells a client which of those has happened. Probing until the answer +/// changes is the only mechanism available, and the short TTL is what makes it work - several attempts fit +/// inside the window. +/// +/// +/// The rule is "take any address that is not the one being retired". Note it is deliberately *not* "prefer an +/// address that has newly appeared", even though a MOVING implies one exists - it is emitted when the +/// connection's own proxy *leaves* the endpoint's address set **and** the set gains a member (established +/// across thirteen observations: a pure reduction takes the proxy away without announcing anything, and a pure +/// widening adds addresses while leaving the connection's proxy in place, which is equally silent). A live +/// sibling is at least as good a target as a newly joined node and is available *now*, whereas the new node +/// only becomes visible when the record updates - which can be after the socket has already closed. Preferring +/// the newcomer would mean waiting for it, and waiting is the thing to avoid. The one case where it would pay +/// is a rolling operation, where the sibling we step to may take its own turn later; that costs one further +/// handoff, bounded at one per connection per operation, which is cheaper than a lost window. +/// +/// +/// This rule is also doing more work than it looks. A Redis Cloud hostname usually carries several A records - measured 2026-08-28: 2 for +/// all-nodes, 3 for all-master-shards, 1 for single, all on a 5s TTL - and the count +/// follows actual proxy *placement* rather than the policy name, so a multi-proxy database whose shards happen +/// to share a node still resolves to one address. With several records the first resolution already names a +/// live sibling proxy, so this returns immediately and steps sideways rather than waiting: any proxy of the +/// same database serves the same data, so a sibling now beats the replacement in nine seconds. The poll only +/// engages when the record names nothing but the address being retired - the single-address case, which is +/// exactly where waiting is the only option available. +/// +/// +/// Deliberately free of jitter, clocks-by-configuration and connection state, so that it can be tested +/// exhaustively without a server: the caller supplies the resolver, the interval and the budget. Jitter +/// belongs at the call site, where the existing refresh jitter lives. +/// +/// +internal static class AdvertisedAddressProbe +{ + /// + /// Ordinary DNS, which is what everything but a test uses. + /// + internal static Task DefaultResolveAsync(string host, CancellationToken cancellationToken) => +#if NET + Dns.GetHostAddressesAsync(host, cancellationToken); +#else + Dns.GetHostAddressesAsync(host); +#endif + + /// + /// Polls DNS until it stops naming , or until the window runs out. + /// + /// + /// The replacement endpoint, or null if the window expired without DNS moving - a measured outcome + /// rather than a failure (see the type remarks: DNS has been seen updating three seconds after the socket + /// closed). The caller should then do nothing proactive: the server closes the socket, the reconnect that + /// follows re-resolves anyway because the endpoint is a hostname, and the relaxed timeout window covers + /// the gap. Guessing an address here would be strictly worse. + /// + internal static async Task ProbeAsync( + DnsEndPoint endpoint, + IPAddress retiring, + TimeSpan window, + TimeSpan pollInterval, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (retiring is null) throw new ArgumentNullException(nameof(retiring)); + if (resolve is null) throw new ArgumentNullException(nameof(resolve)); + + // A non-positive window is not an error: a notification can arrive with nothing left of its budget + // (the shard notifications legitimately carry zero or negative times), and "act now" means one attempt + // rather than none. + var deadline = Environment.TickCount + (int)Math.Max(window.TotalMilliseconds, 0); + int attempt = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + attempt++; + + IPAddress[]? addresses = null; + try + { + addresses = await resolve(endpoint.Host, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // a DNS blip mid-handoff is exactly when we can least afford to give up; keep trying until + // the window says otherwise + log?.Invoke($"MOVING: resolve attempt {attempt} for {endpoint.Host} failed: {ex.Message}"); + } + + if (addresses is not null) + { + IPAddress? candidate = null; + bool retiringStillAdvertised = false; + foreach (var address in addresses) + { + if (address.Equals(retiring)) retiringStillAdvertised = true; + else candidate ??= address; + } + + if (candidate is not null) + { + // Two operationally different outcomes, worth distinguishing in a log somebody reads while + // debugging a handoff: we either stepped sideways to a proxy that was already there, or the + // record itself has moved on to the replacement. + log?.Invoke(retiringStillAdvertised + ? $"MOVING: {endpoint.Host} still advertises {retiring}; moving to sibling {candidate} (attempt {attempt})" + : $"MOVING: {endpoint.Host} now resolves to {candidate} after {attempt} attempt(s)"); + return new IPEndPoint(candidate, endpoint.Port); + } + + log?.Invoke($"MOVING: {endpoint.Host} still resolves only to {retiring} (attempt {attempt}); not yet updated"); + } + + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining <= 0) + { + log?.Invoke($"MOVING: {endpoint.Host} never stopped resolving to {retiring} within the window"); + return null; + } + + // never sleep past the deadline: the last attempt should land inside the window, not after it + var delay = (int)Math.Min(pollInterval.TotalMilliseconds, remaining); + if (delay > 0) await Task.Delay(delay, cancellationToken).ForAwait(); + } + } + + /// + /// Whether the address we are connected on is still one of the addresses the hostname advertises. + /// + /// + /// true if still advertised, false if the record no longer names it, and null if DNS + /// could not be asked - which is *not* the same as "no": a resolution failure is no reason to give up a + /// working connection. + /// + /// + /// The trigger that needs no notification, and the reason it matters is a measured gap. On a multi-proxy + /// database, taking a node out for maintenance announced only the data-movement pair - MIGRATING at + /// +4.3s and MIGRATED at +16.6s, to every proxy - and then dropped the victim from DNS at +21.4s and + /// closed its socket *silently* at +34.7s, while sibling connections stayed up past +90s. No + /// MOVING was ever sent. + /// + /// So for thirteen seconds the condition was plainly visible to anybody who asked - our address is no + /// longer advertised - and the client's only other signal was the socket dying with no explanation. Asking + /// this question when a MIGRATED arrives converts that into a controlled handoff. Note + /// MIGRATED is also the notification the server *retains* and replays on connect, so a client that + /// arrives mid-operation gets the same prompt. + /// + /// + /// The same condition is what a support case turned on: a client kept dialling endpoints that no longer + /// existed for 37 hours, because nothing it could observe told it to stop. Two unrelated failures, one + /// detection rule. + /// + /// + internal static async Task IsStillAdvertisedAsync( + DnsEndPoint endpoint, + IPAddress current, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (current is null) throw new ArgumentNullException(nameof(current)); + if (resolve is null) throw new ArgumentNullException(nameof(resolve)); + + IPAddress[] addresses; + try + { + addresses = await resolve(endpoint.Host, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + log?.Invoke($"{endpoint.Host}: cannot tell whether {current} is still advertised: {ex.Message}"); + return null; + } + + // an empty answer is "cannot tell" rather than "no": a record that momentarily resolves to nothing is + // a DNS problem, not an instruction to abandon a connection that is working + if (addresses is null || addresses.Length == 0) + { + log?.Invoke($"{endpoint.Host}: resolved to nothing; treating {current} as still advertised"); + return null; + } + + foreach (var address in addresses) + { + if (address.Equals(current)) return true; + } + + log?.Invoke($"{endpoint.Host}: no longer advertises {current}; it is being taken out of service"); + return false; + } +} diff --git a/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs b/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs new file mode 100644 index 000000000..7a9b7d950 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/ClusterSlotMigration.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// One entry from a cluster slot-migration notification: some slots have moved, or are moving, from one node +/// to another. +/// +/// +/// A single SMIGRATED describes several of these at once, and not necessarily any involving the node +/// that sent it - every node in the cluster reports the same movements, so the sender is not implicitly the +/// source. +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public readonly struct ClusterSlotMigration +{ + internal ClusterSlotMigration(EndPoint? source, EndPoint? target, IReadOnlyList slots, string? raw) + { + Source = source; + Target = target; + Slots = slots; + RawSlots = raw; + } + + /// + /// The node the slots are moving from, or null if the server named it in a form that cannot be + /// dialled. + /// + public EndPoint? Source { get; } + + /// + /// The node the slots are moving to, or null if the server named it in a form that cannot be + /// dialled. + /// + public EndPoint? Target { get; } + + /// + /// The slots that are moving. + /// + public IReadOnlyList Slots { get; } + + /// + /// The slots exactly as the server expressed them, for the cases the parsed form does not survive - a + /// malformed list leaves empty and this populated. + /// + public string? RawSlots { get; } + + /// + public override string ToString() => $"{Format.ToString(Source)} -> {Format.ToString(Target)}: {RawSlots}"; +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs b/src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs new file mode 100644 index 000000000..b3c8ba459 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceEndpointTypeResolver.cs @@ -0,0 +1,90 @@ +using System; +using System.Net; +using System.Net.Sockets; + +namespace StackExchange.Redis.Maintenance; + +/// +/// Chooses which moving-endpoint-type to ask a server for. +/// +/// +/// Two questions, answered independently. *Scope* comes from the address we are actually connected to: a private +/// or otherwise reserved address means we are inside the deployment's network and want the internal forms. +/// *Form* comes from whether the connection is encrypted: TLS implies the FQDN variants, because a certificate +/// generally cannot be validated against a bare address - so a client handed an IP mid-handoff would be unable +/// to verify the endpoint it was told to move to. +/// +/// +/// | private/reserved | otherwise +/// TLS off | internal-ip | external-ip +/// TLS on | internal-fqdn | external-fqdn +/// +/// +/// +/// Classifying the *connected* address matters, rather than the configured endpoint: the latter is usually a +/// hostname, and what decides whether we are inside the network is where it resolved to. Where there is no +/// socket address at all - a tunnel, a custom transport, a Unix domain socket - the honest answer is +/// : we cannot classify, so we ask for no address and reconnect the +/// way we originally connected. +/// +/// +internal static class MaintenanceEndpointTypeResolver +{ + /// + /// Derives the endpoint type for a connection. + /// + internal static MaintenanceEndpointType Derive(IPAddress? connectedAddress, bool isEncrypted) => + connectedAddress is null + ? MaintenanceEndpointType.None + : IsPrivateOrReserved(connectedAddress) + ? (isEncrypted ? MaintenanceEndpointType.InternalFqdn : MaintenanceEndpointType.InternalIp) + : (isEncrypted ? MaintenanceEndpointType.ExternalFqdn : MaintenanceEndpointType.ExternalIp); + + /// + /// Whether an address is private, or otherwise not routable on the public internet. + /// + /// + /// Covers RFC1918 (10/8, 172.16/12, 192.168/16), loopback, IPv4 link-local (169.254/16), IPv6 unique-local + /// (fc00::/7), IPv6 loopback and link-local, and IPv4-mapped IPv6 - which has to be unwrapped first, or an + /// address like ::ffff:10.0.0.1 classifies as public. + /// + /// CGNAT (100.64/10) is deliberately *not* treated as private. It is a genuine judgement call: it is not + /// publicly routable, but a client behind it is not inside the deployment's network either, which is the + /// question being asked here. Worth confirming against how the server classifies it before changing. + /// + /// + internal static bool IsPrivateOrReserved(IPAddress address) + { + if (address is null) throw new ArgumentNullException(nameof(address)); + + // unwrap ::ffff:a.b.c.d, so the IPv4 rules below actually apply to it + if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); + + if (IPAddress.IsLoopback(address)) return true; + + if (address.AddressFamily == AddressFamily.InterNetwork) + { + var bytes = address.GetAddressBytes(); + return bytes[0] switch + { + 10 => true, // 10.0.0.0/8 + 172 => bytes[1] >= 16 && bytes[1] <= 31, // 172.16.0.0/12 + 192 => bytes[1] == 168, // 192.168.0.0/16 + 169 => bytes[1] == 254, // 169.254.0.0/16, link-local + _ => false, + }; + } + + if (address.AddressFamily == AddressFamily.InterNetworkV6) + { + // IsIPv6UniqueLocal only exists from .NET 6, and this library targets down to net461 - so fc00::/7 + // is tested by hand. IsIPv6LinkLocal and IsIPv6SiteLocal are available everywhere. + if (address.IsIPv6LinkLocal || address.IsIPv6SiteLocal) return true; + + var v6 = address.GetAddressBytes(); + if ((v6[0] & 0xFE) == 0xFC) return true; // fc00::/7, unique-local + } + + return false; + } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs new file mode 100644 index 000000000..fbcae6149 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs @@ -0,0 +1,55 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +/// +/// Carries maintenance context on the faults a disruption can cause, so that a timeout during an announced +/// migration is distinguishable from an ordinary one. +/// +/// +/// This follows the established pattern on these types - Commandstatus and Flags on +/// , FailureType on - rather +/// than introducing an exception type nobody is catching yet. Both properties are named for the role they +/// play, not for the type, matching FailureType. +/// +public sealed partial class RedisTimeoutException +{ + /// + /// The maintenance notification in force when this timed out, or + /// if the server had not announced anything. + /// + /// + /// A value here says the server told us to expect disruption, and that the timeout you are looking at + /// happened inside that window - including the tail after the disruption reported completion. It does not + /// promise that the maintenance *caused* the timeout, only that the two coincided; that is still the most + /// useful thing to know when reading a log after the fact. + /// + /// "Inside that window" is deliberately generous at the trailing edge. Timeouts are raised by a + /// once-a-second sweep rather than at the instant they expire, and a command that timed out had already + /// been waiting for its whole timeout before that - so a window covering the command's entire life can + /// have closed before anybody built this exception. A window that closed no longer ago than the command + /// could have been waiting therefore still counts, which catches every genuine case at the cost of + /// occasionally naming a window that a *different*, later command merely followed. + /// + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationType MaintenanceType { get; internal init; } +} + +/// +public sealed partial class RedisConnectionException +{ + /// + /// The maintenance notification in force when this connection faulted, or + /// if the server had not announced anything. + /// + /// + /// Present on this type as well as on because a handoff that misses + /// its deadline surfaces either way round: as a timeout when the command was already in flight, and as a + /// connection fault when the endpoint went away underneath it. + /// + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + public MaintenanceNotificationType MaintenanceType { get; internal init; } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs new file mode 100644 index 000000000..960b501d0 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceHandoff.cs @@ -0,0 +1,159 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Maintenance; + +/// +/// What to do about a MOVING notification. +/// +internal enum HandoffAction +{ + /// Nothing useful is available; let the server close the socket and reconnect then. + None, + + /// + /// Replace the connections once half the announced window has passed, without looking for a better target. + /// + /// + /// The contract's rule for a notification that names no replacement: "schedule a graceful reconnect to its + /// currently configured endpoint after *half* of the grace period is over" - not immediately, and not at the + /// deadline. Used only where there is nothing better to go on, because measurement shows it is premature + /// when there is: at half of a 15s window, DNS had moved in one of three observed runs, so reconnecting on + /// the clock alone would usually land back on the node being retired. + /// + /// Where it *is* right: an address endpoint, or a connection whose address we cannot see. Those cannot be + /// re-resolved, so a change is undetectable from here - but the address may well be a stable front for a + /// backend that has already moved, which is exactly the case the rule was written for. Waiting passively + /// instead means being closed mid-command rather than choosing the moment. + /// + /// + RecycleAtHalfWindow, + + /// Drop our connections so they re-establish against the replacement address. + Recycle, + + /// + /// The server named where to go; point the next connection at it. + /// + /// + /// Not "add an endpoint and retire this one": the endpoint keeps its identity and its TLS host, and only + /// the socket target changes. See ServerEndPoint.HandoffTarget. + /// + MoveTo, +} + +/// +/// The outcome of deciding how to hand off, and why - the reason is logged either way. +/// +internal readonly struct HandoffDecision(HandoffAction action, EndPoint? target, string reason) +{ + public HandoffAction Action { get; } = action; + + public EndPoint? Target { get; } = target; + + public string Reason { get; } = reason; + + public override string ToString() => Target is null ? $"{Action}: {Reason}" : $"{Action} -> {Target}: {Reason}"; +} + +/// +/// Decides what a MOVING notification means for this connection, and where to go. +/// +/// +/// Separated from the acting so that it can be tested exhaustively without a server: the caller supplies the +/// endpoint, the address it is currently on, the announced window and a resolver, and gets back an action. +/// +/// The dispatch turns on the *form* of the endpoint, which is the thing that decides whether a handoff is even +/// possible - and the answer corrects an earlier assumption that MOVING should reuse the endpoint +/// retirement path: +/// +/// +/// +/// A hostname endpoint with no named successor - the case every observed MOVING has been - keeps its +/// : only the address behind the name moves. Retiring the endpoint would delete our +/// only way of reaching the deployment. So the action is to recycle the connections once DNS has moved, which +/// re-resolves them. +/// +/// +/// An address endpoint with a named successor is genuinely a different endpoint, so the topology has to be +/// re-read. Never observed: eleven routes have all carried an explicit null, including cases where the server +/// had already chosen the replacement node. Treat this branch as code that must exist rather than code that is +/// exercised. +/// +/// +/// An address endpoint with no successor has nothing to re-resolve and nowhere named to go. Doing nothing is +/// correct: the socket closes, the reconnect happens, and the relaxed window covers it. +/// +/// +/// +internal static class MaintenanceHandoff +{ + internal static async Task DecideAsync( + EndPoint endpoint, + EndPoint? successor, + IPAddress? currentAddress, + TimeSpan window, + TimeSpan pollInterval, + Func> resolve, + Action? log = null, + CancellationToken cancellationToken = default) + { + if (successor is not null) + { + // Straight there: no DNS involved, which is the whole value of the field. Measured on RS 8.0.22, + // DNS trails a MOVING by 4.4s to 18.7s while the socket closes at 15.7s to 19.1s, so a named + // successor is the difference between moving immediately and possibly not moving in time at all. + return new HandoffDecision(HandoffAction.MoveTo, successor, "the server named a replacement endpoint"); + } + + if (endpoint is not DnsEndPoint dns) + { + return new HandoffDecision( + HandoffAction.RecycleAtHalfWindow, + null, + $"{Format.ToString(endpoint)} is an address, not a name, and no replacement was named: nothing to re-resolve"); + } + + if (currentAddress is null) + { + // Without knowing where we are, "has it moved" is unanswerable, so there is nothing to poll for - + // which is precisely when the contract's half-window reconnect is the right tool. + return new HandoffDecision( + HandoffAction.RecycleAtHalfWindow, null, "the address of the current connection is unknown"); + } + + var replacement = await AdvertisedAddressProbe.ProbeAsync( + dns, currentAddress, window, pollInterval, resolve, log, cancellationToken).ForAwait(); + + return replacement is null + ? new HandoffDecision( + HandoffAction.None, + null, + $"{dns.Host} still resolved only to {currentAddress} when the window expired") + : new HandoffDecision( + HandoffAction.Recycle, + replacement, + $"{dns.Host} now resolves to {replacement}"); + } + + /// + /// How long to wait before probing, so a fleet does not resolve in lockstep. + /// + /// + /// A *fraction* of the announced window rather than a fixed delay, which is the difference from the refresh + /// jitter elsewhere. Windows are not always generous - the shard notifications have been measured announcing + /// two seconds - so a flat one-second jitter could spend half of one, and on a short window the right amount + /// of jitter is almost none. Capped as well as scaled, because a long window does not justify a long wait + /// when DNS has been seen moving after four seconds. + /// + internal static TimeSpan GetJitter(TimeSpan window, Random random) + { + if (window <= TimeSpan.Zero) return TimeSpan.Zero; + + var tenth = window.TotalMilliseconds / 10; + var capped = Math.Min(tenth, 1000); + return TimeSpan.FromMilliseconds(random.Next(0, (int)Math.Max(capped, 1))); + } +} diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs b/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs new file mode 100644 index 000000000..ae66e4ce2 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/MaintenanceNotificationType.cs @@ -0,0 +1,58 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// The kind of a server-native maintenance notification. +/// +/// +/// Two families share this enum: the Enterprise proxy notifications ( through +/// ) and the OSS cluster ones (, +/// ). Unrecognized types are dropped rather than surfaced, so no member here means +/// "something we didn't understand". +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceNotificationType +{ + /// + /// Not a maintenance notification; the default, for contexts where this is simply not applicable. + /// + None = 0, + + /// + /// This endpoint is being replaced; the notification names its successor, or names nothing at all if the + /// server has no address to offer. + /// + Moving, + + /// + /// A shard is migrating away from this node. Expect latency; expect a after it. + /// + Migrating, + + /// + /// A migration announced by has completed. + /// + Migrated, + + /// + /// This node is failing over. Expect latency; expect a after it. + /// + FailingOver, + + /// + /// A failover announced by has completed. + /// + FailedOver, + + /// + /// Slots are migrating (the OSS cluster family). + /// + SlotMigrating, + + /// + /// Slots announced by have migrated. + /// + SlotMigrated, +} diff --git a/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs new file mode 100644 index 000000000..67d2ac955 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/PushMaintenanceEvent.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using RESPite; + +namespace StackExchange.Redis.Maintenance; + +/// +/// A server-native maintenance notification, received as a RESP3 push frame on the connection that carries +/// commands. One class with a discriminator rather than a type per +/// notification: the payloads are near-identical, and it keeps the handling in one place. +/// +/// +/// The client acts on these itself - relaxing timeouts for the duration, learning a new topology, and moving +/// off an endpoint that says it is going away - so an application that only watches is watching work that has +/// already happened. See the ServerMaintenanceEvent documentation for what each notification does. +/// +/// One notification is raised once, however many nodes announced it: every node broadcasts a given event, so +/// the copies are collapsed on their sequence number and names whichever node arrived +/// first. And one case is deliberately *not* raised: a completion that the server retained and replayed to a +/// connection opting in later. That is history rather than news - it carries no time, so its age is +/// unknowable - and it is recorded in the log instead of being handed to a consumer who could only guess at +/// what to do with it. +/// +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public sealed class PushMaintenanceEvent : ServerMaintenanceEvent +{ + internal PushMaintenanceEvent( + MaintenanceNotificationType notificationType, + long sequenceId, + EndPoint? endPoint, + TimeSpan? time, + EndPoint? newEndPoint, + string? payload, + string rawMessage, + IReadOnlyList? slotMigrations = null) + { + NotificationType = notificationType; + SequenceId = sequenceId; + EndPoint = endPoint; + Time = time; + NewEndPoint = newEndPoint; + Payload = payload; + RawMessage = rawMessage; + SlotMigrations = slotMigrations ?? []; + if (time is { } value && value > TimeSpan.Zero) + { + StartTimeUtc = ReceivedTimeUtc + value; + } + } + + /// + /// Which notification this is. + /// + public MaintenanceNotificationType NotificationType { get; } + + /// + /// The sequence number the server attached to this notification. + /// + /// + /// These identify the *event* rather than the connection that delivered it, which is what makes them + /// useful: every node that broadcasts a given event carries the same value, so a notification arriving + /// twice - once per proxy, or replayed on a reconnect - is recognisable as the same one. The client uses + /// them for exactly that, and so can you. + /// + /// They are allocated per database, ascending, and shared across notification types, so a + /// at 16 is followed by its + /// at 17. Best used for correlating and + /// de-duplicating notifications rather than as an arithmetic sequence: gaps are normal, since a client + /// only sees the events relevant to it. + /// + /// + public long SequenceId { get; } + + /// + /// The server that sent the notification. + /// + /// + /// More precisely: whichever node told us first. Every node broadcasts a given event, so a three-proxy + /// deployment delivers one migration three times, on three connections, with the same + /// . All three are acted on internally - the timeout relaxation is per-server, so + /// each connection genuinely has to see it - but the event is raised once, for the first arrival, and the + /// rest are dropped. Do not read this as "the node being maintained": for the cluster notifications it is + /// usually a bystander reporting someone else's movements (see ). + /// + public EndPoint? EndPoint { get; } + + /// + /// How long the announced event is expected to take, or how much of it remains. + /// + /// + /// The meaning is per-notification: for it is the budget + /// for completing the move, and for the others it is the remaining duration of the announced disruption. + /// It can legitimately be zero or negative - a connection that arrives mid-window is told what is left of + /// it - which means "act now" rather than being an error. null where the notification carries no + /// time at all. + /// + public TimeSpan? Time { get; } + + /// + /// For , the endpoint this one is being replaced by. + /// + /// + /// null is a documented outcome, not just a parse failure: a server with no address to offer sends + /// an explicit null, and it also does so when it cannot honour the endpoint type that was requested. The + /// intended handling in that case is to reconnect using the endpoint already configured, rather than to + /// treat the notification as invalid. See for what actually arrived. + /// + public EndPoint? NewEndPoint { get; } + + /// + /// The final element of the notification, as received: the affected shard ids, the affected slots, or the + /// endpoint of a . + /// + /// + /// Deliberately opaque. Nothing a client is asked to *do* depends on which shards are involved, so this is + /// carried through for diagnostics rather than parsed into a model that the contract does not pin down. + /// + public string? Payload { get; } + + /// + /// For the cluster notifications, the slot movements described; empty for everything else. + /// + /// + /// A notification carries several of these, and the node that sent it is not necessarily the source of + /// any of them - every node reports the same movements. + /// + public IReadOnlyList SlotMigrations { get; } + + /// + public override string? ToString() => RawMessage; +} diff --git a/src/StackExchange.Redis/MaintenanceEndpointType.cs b/src/StackExchange.Redis/MaintenanceEndpointType.cs new file mode 100644 index 000000000..8f0639e1c --- /dev/null +++ b/src/StackExchange.Redis/MaintenanceEndpointType.cs @@ -0,0 +1,64 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis; + +/// +/// Which form of address a server should name when it tells us an endpoint is moving. +/// +/// +/// Sent as the moving-endpoint-type parameter of the maintenance-notification opt-in, and it decides what +/// arrives in . +/// +/// Worth asking for rather than leaving to the server. Eleven observed MOVING notifications on Redis +/// Enterprise 8.0.22 all carried an explicit null, including ones where the server had already chosen the +/// replacement node - and every one of those was requested with a bare ON, so the working theory is that +/// the server default amounts to and we were getting what we asked for. +/// +/// +/// The choice is not cosmetic where TLS is involved: a certificate that carries DNS names and no IP SAN cannot +/// validate an address, so a verifying client that is handed an IP cannot use it. Prefer the FQDN forms when +/// connecting with TLS. +/// +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceEndpointType +{ + /// + /// Do not ask; let the server choose. This is the default, and matches what the client has always sent. + /// + /// + /// In practice this has been observed to mean "no address at all", so a client that wants a named + /// replacement should ask for one explicitly. + /// + ServerDefault = 0, + + /// + /// Work out the right form per connection, and ask for it. The recommended setting. + /// + /// + /// Derived from two facts about the connection as established: whether the address we actually reached is + /// private (so we want the internal forms) and whether the connection is encrypted (so we want the FQDN + /// forms, since a certificate generally cannot be validated against a bare address). Where there is no + /// socket address to classify - a tunnel, or a Unix domain socket - this resolves to + /// rather than guessing. + /// + Auto, + + /// A private address, for a client inside the deployment's network. + InternalIp, + + /// A private hostname, for a client inside the deployment's network. + InternalFqdn, + + /// A public address. Note an address cannot be validated against a DNS-only certificate. + ExternalIp, + + /// A public hostname. The right choice when connecting with TLS. + ExternalFqdn, + + /// + /// Explicitly ask for no address, so a handoff always goes back through the endpoint as configured. + /// + None, +} diff --git a/src/StackExchange.Redis/MaintenanceNotificationMode.cs b/src/StackExchange.Redis/MaintenanceNotificationMode.cs new file mode 100644 index 000000000..fc7c15623 --- /dev/null +++ b/src/StackExchange.Redis/MaintenanceNotificationMode.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis; + +/// +/// Whether to ask a server to send maintenance notifications - advance warning of migrations, failovers and +/// endpoint moves. +/// +/// +/// The three modes, and their names, are prescribed cross-client so that configuration and support tickets +/// line up between clients. Only Redis Enterprise and Redis Cloud emit these notifications; OSS Redis, Valkey +/// and Garnet do not recognize the opt-in at all, which is why exists and why the default +/// is rather than asking every server in existence. +/// +/// Note that means "required", not "on": it rejects any connection that cannot +/// deliver notifications. is the mode that turns the feature on where available and +/// stays out of the way otherwise, and is what most callers want. The names are the cross-client ones, and +/// this is the one place they are easy to misread. +/// +/// +[Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] +public enum MaintenanceNotificationMode +{ + /// + /// Never ask; no notification is requested and none will arrive. + /// + Disabled = 0, + + /// + /// Requires them, and REJECTS THE CONNECTION if they are unavailable - including against any + /// server that does not support them, and on any RESP2 connection. Only for a deployment known to + /// support them; use to ask without that risk. + /// + /// + /// A server that refuses fails the connection, and so does a server that answers HELLO 3 as RESP2, + /// since nothing can be delivered on a RESP2 connection. So does a configuration that could never ask in + /// the first place - Protocol = Resp2, or HELLO unavailable - because requiring a + /// RESP3-only feature over RESP2 is a contradiction, and honouring half of it silently is the outcome + /// this mode exists to prevent. + /// + Enabled, + + /// + /// Ask, and carry on if the server refuses or the connection ends up RESP2 - the feature is then simply + /// off for that server, and the connection is never rejected over it. Safe against a mixture of servers, + /// or one whose support you don't know. + /// + Auto, +} diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 837651106..c6aceb60b 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -62,7 +62,10 @@ internal const CommandFlags NoFlushFlag = (CommandFlags)1024, // "server specific" (bit 18): tied to a specific endpoint, never retry elsewhere. Not (yet) a // public CommandFlags member - see the note on the hidden bit-18 value in CommandFlags.cs. - CommandServerSpecific = (CommandFlags)(1 << 18); + CommandServerSpecific = (CommandFlags)(1 << 18), + // "probe" (bit 19): health-check traffic. Deliberately *not* InternalCallFlag, which also decides + // queuing - see IsCallerFacing. + ProbeFlag = (CommandFlags)(1 << 19); protected RedisCommand command; @@ -92,7 +95,15 @@ internal const CommandFlags | CommandFlags.NoScriptCache | MaskRetryCategory // caller may override the retry category... | CommandServerSpecific // ...and the server-specific flag - | NoFlushFlag; // we'll allow this one even though not advertised + | NoFlushFlag // we'll allow this one even though not advertised + // ...and the probe flag, which *has* to survive this + // whitelist: health-check probes reach the pipeline + // through the public API (HealthCheckContext. + // ProbeFlags), so it arrives as caller-supplied flags + // or not at all. A caller passing it deliberately only + // opts their own command out of endpoint-idleness + // accounting, which is harmless. + | ProbeFlag; private IResultBox? resultBox; @@ -271,6 +282,32 @@ internal void WithHighIntegrity(uint value) public bool IsFireAndForget => (Flags & CommandFlags.FireAndForget) != 0; public bool IsInternalCall => (Flags & InternalCallFlag) != 0; + /// + /// Whether this is health-check traffic, issued by a + /// rather than by a caller. + /// + public bool IsProbe => (Flags & ProbeFlag) != 0; + + /// + /// Whether somebody outside the library is waiting on this message. + /// + /// + /// The question idleness actually wants to ask. Our own traffic - handshakes, heartbeats, + /// autoconfigure, health probes - is work we chose to do, so counting it makes a server look busy + /// precisely because we are looking at it, and an endpoint that can never be retired is the result + /// (see the endpoint-retirement work). + /// + /// A probe is a separate bit from on purpose. The internal-call flag also + /// decides *queuing*: an internal call bypasses the backlog + /// (PhysicalBridge.TryPushToBacklog) and is queued while disconnected regardless of the backlog + /// policy (QueueOrFailMessage). Bypassing the backlog would make a health check unable to + /// observe the one thing it most needs to - a bridge whose queue is not draining - and that signal is + /// what drives failover in a geo-redundant deployment. So probes are excluded from *accounting* without + /// being given internal-call *routing*. + /// + /// + public bool IsCallerFacing => (Flags & (InternalCallFlag | ProbeFlag)) == 0; + public IResultBox? ResultBox => resultBox; public abstract int ArgCount { get; } // note: over-estimate if necessary diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 9baa29d20..4a3dbd0ae 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -390,6 +390,14 @@ internal void KeepAlive(bool forceRun = false) msg.SetSource(ResultProcessor.Tracer, null); break; case ConnectionType.Subscription: + // Normally the PING - observed against a 7.0 server, answered with the two-element array + // pong rather than +PONG (OnResponseFrame's IsArrayPong is what keeps that out of the + // out-of-band path so it still matches this message). The UNSUBSCRIBE fallback is not just + // for pre-3.0 servers: the condition also fails when PING is disabled or renamed in the + // CommandMap, or fronted by something that does not support it. + // + // Neither needs SetInternalCall here: the common path below flags whatever the switch + // produced, so any tracer added later is covered without remembering to. if (commandMap.IsAvailable(RedisCommand.PING) && features.PingOnSubscriber) { msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.PING); @@ -499,9 +507,20 @@ internal void OnDisconnected(ConnectionFailureType failureType, PhysicalConnecti } ServerEndPoint.OnDisconnected(this); - if (!isDisposed && Interlocked.Increment(ref failConnectCount) == 1) + if (!isDisposed) { - TryConnect(null); // try to connect immediately + var consecutive = Interlocked.Increment(ref failConnectCount); + if (consecutive == 1) + { + TryConnect(null); // try to connect immediately + } + + // An endpoint we cannot even connect to is evidence that what we believe about the + // deployment may be wrong - and until now nothing acted on that. The reconfigure-on-failure + // path is gated on reconfigureNextFailure, which is only ever set once a connection has + // been *established*, so a node that has only ever refused could be retried forever + // without anybody re-reading the topology. Measured in the field: 37 hours. + ServerEndPoint.OnRepeatedConnectFailure(consecutive); } } else if (physical == null) @@ -673,8 +692,12 @@ internal void OnHeartbeat(bool ifConnectedOnly) // This is an "always" check - we always want to evaluate a dead connection from a non-responsive sever regardless of the need to heartbeat above var totalTimeoutThisHeartbeat = asyncTimeoutThisHeartbeat + syncTimeoutThisHeartbeat; - bool deadConnectionOnAsync = asyncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds * 4); - bool deadConnectionOnSync = syncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.TimeoutMilliseconds * 4); + // note these track the *effective* timeout: this heuristic is derived from the + // command timeout, so if relaxation says "be patient for 60s" while this says + // "dead after 4x5s", we would tear down the connection relaxation was protecting. + // Socket-level failure detection is untouched and still notices a dead server. + bool deadConnectionOnAsync = asyncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (ServerEndPoint.GetEffectiveTimeoutMilliseconds(tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds ?? 0) * 4); + bool deadConnectionOnSync = syncTimeoutThisHeartbeat > 0 && tmp.LastReadSecondsAgo * 1_000 > (ServerEndPoint.GetEffectiveTimeoutMilliseconds(tmp.BridgeCouldBeNull?.Multiplexer.TimeoutMilliseconds ?? 0) * 4); if (deadConnectionOnAsync || deadConnectionOnSync) { // If we've received *NOTHING* on the pipe in 4 timeouts worth of time and we're timing out commands, issue a connection failure so that we reconnect @@ -1019,10 +1042,29 @@ private void StartBacklogProcessor() /// Crawls from the head of the backlog queue, consuming anything that should have timed out /// and pruning it accordingly (these messages will get timeout exceptions). /// + /// + /// Whether this bridge owes a *caller* anything, ignoring our own internal traffic. + /// + internal bool HasCallerWork() + { + if (physical?.HasCallerMessagesAwaitingResponse() == true) return true; + + foreach (var message in _backlog) // snapshot enumeration; a concurrent queue is fine to walk + { + if (message.IsCallerFacing) return true; + } + return false; + } + private void CheckBacklogForTimeouts() { var now = Environment.TickCount; - var timeout = _singleWriter.TimeoutMilliseconds; + + // deliberately *not* _singleWriter.TimeoutMilliseconds, which is the write-lock acquisition + // timeout: that is about contention between writers, and relaxing it during maintenance is not + // something anybody asked for. This is the age at which a queued command is considered timed out, + // relaxed while the server has announced a disruption. + var timeout = ServerEndPoint.GetEffectiveTimeoutMilliseconds(Multiplexer.TimeoutMilliseconds); // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. @@ -1303,7 +1345,7 @@ public bool HasPendingCallerFacingItems() { foreach (var item in _backlog) // non-consuming, thread-safe, etc { - if (!item.IsInternalCall) return true; + if (item.IsCallerFacing) return true; } } return physical?.HasPendingCallerFacingItems() ?? false; @@ -1763,6 +1805,33 @@ internal void SimulateConnectionFailure(SimulatedFailureType failureType) physical?.SimulateConnectionFailure(failureType); } + /// + /// Drops the current connection so that a fresh one is established. + /// + /// + /// Deliberately just a dispose: that routes through RecordConnectionFailed to + /// , which reconnects immediately by itself, so there is no new lifecycle + /// here to get wrong. Used by the MOVING handoff, where the point is to choose *when* the + /// connection is replaced - after the replacement address is known - rather than waiting for the server + /// to close it and re-resolving to whatever DNS says at that moment, which has been measured as still + /// naming the node being retired. + /// + internal bool RecycleConnection(string reason) + { + var current = physical; + if (current is null) return false; + + Multiplexer.Trace($"recycling {ConnectionType} connection: {reason}", ToString()); + + // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises + // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and + // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first + // also performs the disconnect, so the Dispose below is cleanup. + current.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + current.Dispose(); + return true; + } + internal RedisCommand? GetActiveMessage() => Volatile.Read(ref _activeMessage)?.Command; } } diff --git a/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs new file mode 100644 index 000000000..a993b2a69 --- /dev/null +++ b/src/StackExchange.Redis/PhysicalConnection.Maintenance.cs @@ -0,0 +1,346 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using RESPite; +using RESPite.Messages; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +internal sealed partial class PhysicalConnection +{ + /// + /// Reads a maintenance notification and raises it as an event; observation only. + /// + /// + /// Parsed leniently on purpose. These frames are dispatched here *before* anything reads element 1 as a + /// channel name, because element 1 is a sequence number - so a malformed one must be swallowed here, and + /// never fall through to the command matcher, where it would take a reply belonging to something else. + /// The shapes are (per the contract, all elements after the type being scalars): + /// + /// MOVING seq time endpoint + /// MIGRATING seq time shards, FAILING_OVER seq time shards + /// MIGRATED seq shards, FAILED_OVER seq shards + /// SMIGRATING seq slots, SMIGRATED seq slots + /// + /// Integers may arrive as : or as a bulk string, trailing elements are explicitly allowed, and the + /// endpoint may be an explicit null. So the type decides *whether* a time is expected rather than the + /// content deciding it - a slot list of "123" must not be mistaken for a duration - but a notification + /// that omits or adds one is still accepted. + /// + private OutOfBandResult OnMaintenanceNotification(ConnectionMultiplexer muxer, PushKind kind, ref RespReader reader) + { + _readStatus = ReadStatus.MaintenanceNotification; + + // at most three elements follow the type in any defined shape; anything beyond that is ignored + string? e1 = null, e2 = null, e3 = null; + List? migrations = null; + int count = 0; + while (reader.SafeTryMoveNext()) + { + count++; + if (!reader.IsScalar) + { + // SMIGRATED nests: [type, seq, [[source, target, slots], ...]]. Everything else is flat, and + // nesting there means a frame we do not understand - so the guard stays for those, since + // guessing at an unknown shape is how a parser starts inventing data + if (kind is PushKind.SlotMigrated or PushKind.SlotMigrating && migrations is null) + { + // note the reader has to be moved *past* the aggregate: enumerating the children does not + // advance it, so without this the loop walks back into the triplets we just read and + // mistakes them for further top-level elements + migrations = ReadSlotMigrations(ref reader); + continue; + } + + Trace($"{kind}: non-scalar element {count}"); + return OutOfBandResult.Handled; + } + + var element = reader.IsNull ? null : reader.ReadString(); + switch (count) + { + case 1: e1 = element; break; + case 2: e2 = element; break; + case 3: e3 = element; break; + } + } + + var type = ToNotificationType(kind); + + // A missing or unreadable sequence id is *not* fatal. The specs say every shape carries one, but + // go-redis - which runs against real servers - length-checks these frames at two elements and reads + // no sequence number at all for the shard notifications. So a client that drops a frame for want of a + // seq is stricter than one that demonstrably works. Without it we lose only dedup, which is our own + // invention anyway; the disruption being announced is the part that matters. + long? sequenceId = TryParseInt64(e1, out var parsedSequenceId) ? parsedSequenceId : null; + if (sequenceId is null) + { + OnMaintenanceNotificationDropped(type, $"no readable sequence id in a {count + 1}-element frame; continuing without dedup"); + } + + long? timeSeconds = null; + string? payload; + if (CarriesTime(type)) + { + if (TryParseInt64(e2, out var seconds)) + { + timeSeconds = seconds; + payload = e3; + } + else + { + // tolerated: a server that omits the time it was supposed to send + payload = e3 ?? e2; + } + } + else if (count >= 3 && TryParseInt64(e2, out var seconds)) + { + // tolerated the other way: a time on a notification the contract says has none + timeSeconds = seconds; + payload = e3; + } + else + { + payload = e3 ?? e2; + } + + var server = BridgeCouldBeNull?.ServerEndPoint; + EndPoint? newEndPoint = null; + if (type == MaintenanceNotificationType.Moving && !string.IsNullOrEmpty(payload)) + { + // "?" and an empty host are the contract's placeholders for "no address", and are no more + // dialable than the null form; they must never be taken to mean "the server that told us" + newEndPoint = ParseMigrationEndPoint(payload); + if (newEndPoint is null) + { + Trace($"{kind}: no usable endpoint in '{payload}'"); + } + } + + var time = timeSeconds is { } value ? TimeSpan.FromSeconds(value) : (TimeSpan?)null; + var raw = Describe(kind, sequenceId, timeSeconds, payload); + Trace($"maintenance notification: {raw}"); + OnDetailLog($"maintenance notification: {raw}"); + + // A *retained* notification arriving before the bridge reports established is the server's catch-up + // copy: Enterprise keeps the most recent shard-scoped completion and replays it to whoever opts in + // next, with no measured age limit (the same FAILED_OVER came back three hours later). + // + // Both halves of the test are load-bearing. "Before established" is the only signal available, since a + // completion carries no time; and restricting it to the retained kinds is what keeps a *live* + // notification that merely happens to land mid-handshake from being mistaken for history - a + // late-joining connection can legitimately be told about a disruption in progress, and `SMIGRATED` is + // not retained at all, so one arriving here is news. + var isCatchUp = IsRetained(type) && BridgeCouldBeNull?.IsConnected != true; + + // relax before reporting: the event handler is consumer code, and the window should already be open + // by the time anyone sees the notification that opened it + if (server is not null) + { + // Logged, not merely traced: Trace is [Conditional("VERBOSE")], so until now a received + // notification was invisible in an ordinary deployment - including to the log-based verification + // our own documentation recommends. This is the line that answers "why is my new connection + // relaxed?", so it names the catch-up case explicitly. + muxer.Logger?.LogInformationMaintenanceNotificationReceived( + new(server), type, sequenceId ?? -1, isCatchUp ? " (catch-up)" : string.Empty); + + if (IsWindowOpening(type)) + { + var isNew = server.OnMaintenanceWindowOpened(type, sequenceId, time); + + // ...and MOVING alone means "this endpoint is going away", which is worth acting on rather than + // waiting to be disconnected. + // + // Only when the notification is *new*, and that is load-bearing rather than tidy. A server + // re-sends MOVING to a connection that opts in while the window is still open - measured, and + // the handoff replaces the connection, so acting on the repeat is a feedback loop: recycle, + // reconnect, get told again, recycle. It produced twelve recycles from one event on a real + // deployment before this guard. The per-server sequence dedup already knew it was a repeat; the + // handoff simply was not asking. + if (isNew && type == MaintenanceNotificationType.Moving) + { + server.OnMovingAnnounced(time, newEndPoint, this); + } + } + else if (IsWindowClosing(type)) + { + server.OnMaintenanceWindowClosed(type, sequenceId, isCatchUp); + + // ...and if slots moved away from us, learn the new topology rather than waiting to be told + // by a -MOVED. Scoped and jittered inside OnSlotsMigratedAway; see its remarks for why this + // is the cluster family only + if (type == MaintenanceNotificationType.SlotMigrated && migrations is not null) + { + server.OnSlotsMigratedAway(migrations); + } + } + } + + // A catch-up copy is not reported at all. We ignore it internally - it opens no window, because it is + // history rather than news - and raising it anyway would hand a consumer a notification it cannot + // date: nothing in the frame says whether the failover was seconds or hours ago, so any action taken + // on it is a guess. It stays visible in the log (with "(catch-up)" against it), which is the right + // place for "here is what the server mentioned on the way in". + if (isCatchUp) + { + Trace($"{kind} seq {sequenceId} is a retained copy of a finished event; not raising it"); + return OutOfBandResult.Handled; + } + + // Per-server work above, one event below: relaxation is per-connection and every connection is told, + // but a consumer wants one callback per logical event rather than one per proxy that mentioned it + if (muxer.TryClaimMaintenanceEvent(type, sequenceId)) + { + var evt = new PushMaintenanceEvent(type, sequenceId ?? 0, server?.EndPoint, time, newEndPoint, payload, raw, migrations); + muxer.OnServerMaintenanceEvent(evt); + } + else + { + Trace($"{kind} seq {sequenceId} already reported by another node; not raising again"); + } + + return OutOfBandResult.Handled; + } + + /// + /// Reads the nested [[source, target, slots], ...] form of a cluster slot-migration notification. + /// + /// + /// A malformed triplet is skipped rather than abandoning the whole notification - the same choice go-redis + /// makes, and the right one: the other triplets are still actionable, and one bad entry should not lose a + /// migration we could have applied. The slot list is a flat comma-and-range string inside each triplet. + /// + private List ReadSlotMigrations(ref RespReader reader) + { + var results = new List(); + var outer = reader.AggregateChildren(); + while (outer.MoveNext()) + { + var triplet = outer.Value; + if (!triplet.IsAggregate) + { + Trace("slot migration: expected a triplet"); + continue; + } + + string? source = null, target = null, slots = null; + int index = 0; + var inner = triplet.AggregateChildren(); + while (inner.MoveNext()) + { + var child = inner.Value; + var text = child.IsScalar && !child.IsNull ? child.ReadString() : null; + switch (index++) + { + case 0: source = text; break; + case 1: target = text; break; + case 2: slots = text; break; + } + } + + if (index < 3) + { + Trace($"slot migration: {index}-element triplet, skipped"); + continue; + } + + var ranges = SlotRange.TryParseList(slots, out var parsed) ? parsed : []; + results.Add(new ClusterSlotMigration(ParseMigrationEndPoint(source), ParseMigrationEndPoint(target), ranges, slots)); + } + + // leave the caller's reader after the aggregate, so it can carry on with any trailing elements + outer.MovePast(out reader); + return results; + } + + /// + /// Parses one end of a migration, treating the placeholder forms as "not named" rather than as an error. + /// + private static EndPoint? ParseMigrationEndPoint(string? value) + => string.IsNullOrEmpty(value) || value == "?" || !Format.TryParseEndPoint(value, out var parsed) || IsPlaceholderEndPoint(parsed) + ? null + : parsed; + + private static bool IsPlaceholderEndPoint(EndPoint endpoint) => endpoint switch + { + DnsEndPoint dns => dns.Port == 0 || dns.Host is "" or "?", + IPEndPoint ip => ip.Port == 0, + _ => false, + }; + + private void OnMaintenanceNotificationDropped(MaintenanceNotificationType type, string reason) + { + // never fatal: a notification we cannot read is a diagnostic, not a protocol failure - the frame has + // been consumed either way, so the connection is not at risk + Trace($"dropped {type} notification: {reason}"); + OnDetailLog($"dropped {type} notification: {reason}"); + } + + private static bool TryParseInt64(string? value, out long result) + => long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out result); + + /// + /// Whether this notification announces a disruption starting (or still running). + /// + /// + /// is an opener with no closer: its window can only end + /// by the deadline the server gave us, or - later - by the handoff completing. + /// + private static bool IsWindowOpening(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Moving + or MaintenanceNotificationType.Migrating + or MaintenanceNotificationType.FailingOver + or MaintenanceNotificationType.SlotMigrating; + + /// + /// Whether this notification announces that a disruption has finished. + /// + private static bool IsWindowClosing(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.FailedOver + or MaintenanceNotificationType.SlotMigrated; + + /// + /// Whether the server retains this notification and replays it to connections that opt in later. + /// + /// + /// Measured on Redis Enterprise rather than specified: the most recent shard-scoped *completion* is + /// retained, most-recent-replaces, and nothing else is - not the starters, not MOVING, and not the + /// slot-scoped cluster forms. That is what makes a replay unable to demand action, and it is why only + /// these two kinds can arrive as history. + /// + private static bool IsRetained(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.FailedOver; + + /// + /// Whether the contract gives this notification a time element. + /// + private static bool CarriesTime(MaintenanceNotificationType type) => type is + MaintenanceNotificationType.Moving + or MaintenanceNotificationType.Migrating + or MaintenanceNotificationType.FailingOver; + + private static MaintenanceNotificationType ToNotificationType(PushKind kind) => kind switch + { + PushKind.Moving => MaintenanceNotificationType.Moving, + PushKind.Migrating => MaintenanceNotificationType.Migrating, + PushKind.Migrated => MaintenanceNotificationType.Migrated, + PushKind.FailingOver => MaintenanceNotificationType.FailingOver, + PushKind.FailedOver => MaintenanceNotificationType.FailedOver, + PushKind.SlotMigrating => MaintenanceNotificationType.SlotMigrating, + PushKind.SlotMigrated => MaintenanceNotificationType.SlotMigrated, + _ => MaintenanceNotificationType.None, + }; + + private static string Describe(PushKind kind, long? sequenceId, long? timeSeconds, string? payload) + { + var sb = new System.Text.StringBuilder(kind.ToString().ToUpperInvariant()); + sb.Append(" seq=").Append(sequenceId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "?"); + if (timeSeconds is { } seconds) sb.Append(" time=").Append(seconds).Append('s'); + if (payload is not null) sb.Append(' ').Append(payload); + return sb.ToString(); + } +} diff --git a/src/StackExchange.Redis/PhysicalConnection.Read.cs b/src/StackExchange.Redis/PhysicalConnection.Read.cs index e65941021..91efe09c9 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Read.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Read.cs @@ -508,11 +508,36 @@ internal enum PushKind PUnsubscribe, [AsciiHash("sunsubscribe")] SUnsubscribe, + + // the maintenance-notification family; these are *not* pub/sub - element 1 is a sequence number + // rather than a channel, so they must be dispatched before anything reads a channel name. The specs + // write them uppercase while the pub/sub kinds above are lowercase, hence the case-insensitive match + [AsciiHash("MOVING")] + Moving, + [AsciiHash("MIGRATING")] + Migrating, + [AsciiHash("MIGRATED")] + Migrated, + [AsciiHash("FAILING_OVER")] + FailingOver, + [AsciiHash("FAILED_OVER")] + FailedOver, + [AsciiHash("SMIGRATING")] + SlotMigrating, + [AsciiHash("SMIGRATED")] + SlotMigrated, } internal static partial class PushKindMetadata { - [AsciiHash] + /// + /// Identifies a push frame from its first element. + /// + /// + /// Case-insensitive: the pub/sub kinds are lowercase on the wire and the maintenance kinds are + /// uppercase, and no specification anywhere is careful about it - so don't bake in an assumption. + /// + [AsciiHash(CaseSensitive = false)] internal static partial bool TryParse(ReadOnlySpan value, out PushKind result); } @@ -563,6 +588,12 @@ private OutOfBandResult OnOutOfBand(ReadOnlySpan payload, ref IMemoryOwner if (!reader.TryParseScalar(&PushKindMetadata.TryParse, out kind)) kind = PushKind.None; } + if (kind is >= PushKind.Moving and <= PushKind.SlotMigrated) + { + // not pub/sub: dispatch before anything tries to read element 1 as a channel name + return OnMaintenanceNotification(muxer, kind, ref reader); + } + RedisChannel.RedisChannelOptions channelOptions = kind switch { PushKind.PMessage or PushKind.PSubscribe or PushKind.PUnsubscribe => RedisChannel.RedisChannelOptions.Pattern, diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 1c4222584..d5272b28f 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -84,6 +84,19 @@ internal void GetBytes(out long sent, out long received) private Socket? _socket; internal Socket? VolatileSocket => Volatile.Read(ref _socket); + /// + /// Whether this connection is encrypted, however that came about. + /// + /// + /// Two routes, and both count: our own , or a tunnel-supplied transport that + /// reports it is already encrypted. Used to choose a moving-endpoint-type, where the question is + /// not "did the caller ask for TLS" but "is this connection actually encrypted" - because that is what + /// decides whether a certificate has to be validated against whatever address we are given next. + /// + internal bool IsEncrypted => + Volatile.Read(ref _transport)?.IsEncrypted == true + || Volatile.Read(ref _ioStream) is SslStream { IsEncrypted: true }; + // used for dummy test connections public PhysicalConnection( ConnectionType connectionType = ConnectionType.Interactive, @@ -178,6 +191,16 @@ internal async Task BeginConnectAsync(ILogger? log) var rawConfig = bridge.Multiplexer.RawConfig; var tunnel = rawConfig.Tunnel; var connectTo = endpoint; + + // A MOVING may name where to go next; prefer it over resolving the endpoint again, since not + // waiting for DNS is the point. Note this changes only the *socket target*: the endpoint itself is + // untouched, so identity, server selection and - importantly - the TLS host and SNI (derived from + // ServerEndPoint.EndPoint below) all stay exactly as configured. + if (bridge.ServerEndPoint?.HandoffTarget is { } handoffTarget) + { + Trace($"handoff: connecting to {Format.ToString(handoffTarget)} in place of {Format.ToString(endpoint)}"); + connectTo = handoffTarget; + } if (tunnel is not null) { // A transport tunnel replaces the socket outright (the widest form of the existing @@ -811,6 +834,31 @@ internal void GetStormLog(StringBuilder sb) } } + /// + /// Whether any message awaiting a response was issued by a *caller* rather than by us. + /// + /// + /// The distinction matters wherever we ask "is anyone using this server": our own handshake, + /// autoconfigure and keep-alive traffic is not use, and treating it as such makes a server we are + /// probing look busy *because* we are probing it. + /// + /// A predicate rather than a count, because every caller only asks whether the answer is zero - and + /// this way the common case (a caller's message at the head of the queue) costs one iteration instead + /// of walking a queue that a stalled server can leave thousands of entries long. + /// + /// + internal bool HasCallerMessagesAwaitingResponse() + { + lock (_writtenAwaitingResponse) + { + foreach (var message in _writtenAwaitingResponse) + { + if (message.IsCallerFacing) return true; + } + } + return false; + } + /// /// Runs on every heartbeat for a bridge, timing out any commands that are overdue and returning an integer of how many we timed out. /// @@ -829,7 +877,9 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou { var server = bridge.ServerEndPoint; var multiplexer = bridge.Multiplexer; - var timeout = multiplexer.AsyncTimeoutMilliseconds; + + // relaxed while this server has announced a disruption; a floor, never a reduction + var timeout = server.GetEffectiveTimeoutMilliseconds(multiplexer.AsyncTimeoutMilliseconds); foreach (var msg in _writtenAwaitingResponse) { // We only handle async timeouts here, synchronous timeouts are handled upstream. @@ -1272,6 +1322,7 @@ internal enum ReadStatus ResetArena, ProcessBufferComplete, PubSubUnsubscribe, + MaintenanceNotification, NA = -1, } @@ -1287,7 +1338,7 @@ internal bool HasPendingCallerFacingItems() { foreach (var item in _writtenAwaitingResponse) { - if (!item.IsInternalCall) return true; + if (item.IsCallerFacing) return true; } } return false; diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 8022993e4..c7261f397 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,20 +1,19 @@ #nullable enable -StackExchange.Redis.ConfigurationOptions.SentinelPassword.get -> string? -StackExchange.Redis.ConfigurationOptions.SentinelPassword.set -> void -StackExchange.Redis.ConfigurationOptions.SentinelUser.get -> string? -StackExchange.Redis.ConfigurationOptions.SentinelUser.set -> void -[SER009]StackExchange.Redis.Configuration.TlsOptions -[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions() -> void -[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions(StackExchange.Redis.ConfigurationOptions! options) -> void -[SER009]StackExchange.Redis.Configuration.TlsOptions.IsEnabled.get -> bool -[SER009]StackExchange.Redis.Configuration.TlsOptions.SslHost.get -> string? -[SER009]StackExchange.Redis.Configuration.TlsOptions.SslProtocols.get -> System.Security.Authentication.SslProtocols? -[SER009]StackExchange.Redis.Configuration.TlsOptions.CheckCertificateRevocation.get -> bool -[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateValidationCallback.get -> System.Net.Security.RemoteCertificateValidationCallback? -[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateSelectionCallback.get -> System.Net.Security.LocalCertificateSelectionCallback? -[SER009]StackExchange.Redis.Configuration.TlsOptions.ResolveHost(System.Net.EndPoint! endpoint) -> string! -[SER009]virtual StackExchange.Redis.Configuration.Tunnel.ConnectTransportAsync(System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType, StackExchange.Redis.Configuration.TlsOptions tls, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask -StackExchange.Redis.RedisFeatures.Hello.get -> bool +StackExchange.Redis.BitFieldEncoding +StackExchange.Redis.BitFieldEncoding.BitFieldEncoding() -> void +StackExchange.Redis.BitFieldEncoding.Equals(StackExchange.Redis.BitFieldEncoding other) -> bool +StackExchange.Redis.BitFieldEncoding.IsSigned.get -> bool +StackExchange.Redis.BitFieldEncoding.Width.get -> int +StackExchange.Redis.BitFieldOffset +StackExchange.Redis.BitFieldOffset.BitFieldOffset() -> void +StackExchange.Redis.BitFieldOffset.Equals(StackExchange.Redis.BitFieldOffset other) -> bool +StackExchange.Redis.BitFieldOperation +StackExchange.Redis.BitFieldOperation.BitFieldOperation() -> void +StackExchange.Redis.BitFieldOperation.Equals(StackExchange.Redis.BitFieldOperation other) -> bool +StackExchange.Redis.BitFieldOverflow +StackExchange.Redis.BitFieldOverflow.Fail = 2 -> StackExchange.Redis.BitFieldOverflow +StackExchange.Redis.BitFieldOverflow.Saturate = 1 -> StackExchange.Redis.BitFieldOverflow +StackExchange.Redis.BitFieldOverflow.Wrap = 0 -> StackExchange.Redis.BitFieldOverflow StackExchange.Redis.ClusterNode.AuxFields.get -> System.Collections.Generic.IReadOnlyList>! StackExchange.Redis.ClusterNode.ClusterBusPort.get -> int? StackExchange.Redis.ClusterNode.Hostname.get -> string? @@ -32,14 +31,27 @@ StackExchange.Redis.ClusterSlotNode.NodeId.get -> string? StackExchange.Redis.ClusterSlotNode.Port.get -> int StackExchange.Redis.ClusterSlotsResult StackExchange.Redis.ClusterSlotsResult.Assignments.get -> System.Collections.Generic.IReadOnlyList! -StackExchange.Redis.IServer.ClusterSlots(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.ClusterSlotsResult? -StackExchange.Redis.IServer.ClusterSlotsAsync(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.RedisErrorKind.UnknownRedirectTarget = 26 -> StackExchange.Redis.RedisErrorKind -override StackExchange.Redis.ClusterSlotNode.ToString() -> string! -StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.NameValueEntry[]! streamPairs, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue streamField, StackExchange.Redis.RedisValue streamValue, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.Configuration.RedisCloudOptionsProvider +StackExchange.Redis.Configuration.RedisCloudOptionsProvider.RedisCloudOptionsProvider() -> void +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider +StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.RedisEnterpriseOptionsProvider() -> void +StackExchange.Redis.ConfigurationOptions.SentinelPassword.get -> string? +StackExchange.Redis.ConfigurationOptions.SentinelPassword.set -> void +StackExchange.Redis.ConfigurationOptions.SentinelUser.get -> string? +StackExchange.Redis.ConfigurationOptions.SentinelUser.set -> void +StackExchange.Redis.ConnectionFailureType.MaintenanceHandoff = 12 -> StackExchange.Redis.ConnectionFailureType StackExchange.Redis.IDatabase.StreamAdd(StackExchange.Redis.RedisKey key, StackExchange.Redis.NameValueEntry[]! streamPairs, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue StackExchange.Redis.IDatabase.StreamAdd(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue streamField, StackExchange.Redis.RedisValue streamValue, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue +StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long? +StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease! +StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.NameValueEntry[]! streamPairs, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StreamAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue streamField, StackExchange.Redis.RedisValue streamValue, StackExchange.Redis.StreamAddOptions options, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task!>! +StackExchange.Redis.IServer.ClusterSlots(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.ClusterSlotsResult? +StackExchange.Redis.IServer.ClusterSlotsAsync(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.RedisFeatures.BitFieldReadOnly.get -> bool +StackExchange.Redis.RedisFeatures.Hello.get -> bool StackExchange.Redis.StreamAddOptions StackExchange.Redis.StreamAddOptions.Approximate.get -> bool StackExchange.Redis.StreamAddOptions.Approximate.init -> void @@ -59,6 +71,77 @@ StackExchange.Redis.StreamAddOptions.StreamAddOptions() -> void StackExchange.Redis.StreamAddOptions.TrimMode.get -> StackExchange.Redis.StreamTrimMode StackExchange.Redis.StreamAddOptions.TrimMode.init -> void StackExchange.Redis.StringIndex +[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeFlags.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.RedisErrorKind.UnknownRedirectTarget = 26 -> StackExchange.Redis.RedisErrorKind +[SER009]StackExchange.Redis.Configuration.TlsOptions +[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateSelectionCallback.get -> System.Net.Security.LocalCertificateSelectionCallback? +[SER009]StackExchange.Redis.Configuration.TlsOptions.CertificateValidationCallback.get -> System.Net.Security.RemoteCertificateValidationCallback? +[SER009]StackExchange.Redis.Configuration.TlsOptions.CheckCertificateRevocation.get -> bool +[SER009]StackExchange.Redis.Configuration.TlsOptions.IsEnabled.get -> bool +[SER009]StackExchange.Redis.Configuration.TlsOptions.ResolveHost(System.Net.EndPoint! endpoint) -> string! +[SER009]StackExchange.Redis.Configuration.TlsOptions.SslHost.get -> string? +[SER009]StackExchange.Redis.Configuration.TlsOptions.SslProtocols.get -> System.Security.Authentication.SslProtocols? +[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions() -> void +[SER009]StackExchange.Redis.Configuration.TlsOptions.TlsOptions(StackExchange.Redis.ConfigurationOptions! options) -> void +[SER009]virtual StackExchange.Redis.Configuration.Tunnel.ConnectTransportAsync(System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType, StackExchange.Redis.Configuration.TlsOptions tls, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +[SER010]StackExchange.Redis.Availability.FaultContext.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceMovingEndpointType.get -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceMovingEndpointType.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceNotifications.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenancePostEventRelaxedDuration.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedTimeout.set -> void +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.get -> System.TimeSpan +[SER010]StackExchange.Redis.ConfigurationOptions.MaintenanceRelaxedWindowMax.set -> void +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.ClusterSlotMigration() -> void +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.RawSlots.get -> string? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Slots.get -> System.Collections.Generic.IReadOnlyList! +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Source.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.ClusterSlotMigration.Target.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailedOver = 5 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.FailingOver = 4 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrated = 3 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Migrating = 2 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.Moving = 1 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.None = 0 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrated = 7 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.MaintenanceNotificationType.SlotMigrating = 6 -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.EndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NewEndPoint.get -> System.Net.EndPoint? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.NotificationType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Payload.get -> string? +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SequenceId.get -> long +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.SlotMigrations.get -> System.Collections.Generic.IReadOnlyList! +[SER010]StackExchange.Redis.Maintenance.PushMaintenanceEvent.Time.get -> System.TimeSpan? +[SER010]StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.Auto = 1 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalFqdn = 5 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ExternalIp = 4 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalFqdn = 3 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.InternalIp = 2 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.None = 6 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceEndpointType.ServerDefault = 0 -> StackExchange.Redis.MaintenanceEndpointType +[SER010]StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Auto = 2 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Disabled = 0 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.MaintenanceNotificationMode.Enabled = 1 -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]StackExchange.Redis.RedisConnectionException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]StackExchange.Redis.RedisTimeoutException.MaintenanceType.get -> StackExchange.Redis.Maintenance.MaintenanceNotificationType +[SER010]override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]override StackExchange.Redis.Maintenance.ClusterSlotMigration.ToString() -> string! +[SER010]override StackExchange.Redis.Maintenance.PushMaintenanceEvent.ToString() -> string? +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceMovingEndpointType.get -> StackExchange.Redis.MaintenanceEndpointType +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceNotifications.get -> StackExchange.Redis.MaintenanceNotificationMode +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenancePostEventRelaxedDuration.get -> System.TimeSpan? +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedTimeout.get -> System.TimeSpan +[SER010]virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.MaintenanceRelaxedWindowMax.get -> System.TimeSpan? const StackExchange.Redis.StringIndex.Unbounded = -9223372036854775808 -> long override StackExchange.Redis.BitFieldEncoding.Equals(object? obj) -> bool override StackExchange.Redis.BitFieldEncoding.GetHashCode() -> int @@ -69,49 +152,45 @@ override StackExchange.Redis.BitFieldOffset.ToString() -> string! override StackExchange.Redis.BitFieldOperation.Equals(object? obj) -> bool override StackExchange.Redis.BitFieldOperation.GetHashCode() -> int override StackExchange.Redis.BitFieldOperation.ToString() -> string! -StackExchange.Redis.BitFieldEncoding -StackExchange.Redis.BitFieldEncoding.BitFieldEncoding() -> void -StackExchange.Redis.BitFieldEncoding.Equals(StackExchange.Redis.BitFieldEncoding other) -> bool -StackExchange.Redis.BitFieldEncoding.IsSigned.get -> bool -StackExchange.Redis.BitFieldEncoding.Width.get -> int -StackExchange.Redis.BitFieldOffset -StackExchange.Redis.BitFieldOffset.BitFieldOffset() -> void -StackExchange.Redis.BitFieldOffset.Equals(StackExchange.Redis.BitFieldOffset other) -> bool -StackExchange.Redis.BitFieldOperation -StackExchange.Redis.BitFieldOperation.BitFieldOperation() -> void -StackExchange.Redis.BitFieldOperation.Equals(StackExchange.Redis.BitFieldOperation other) -> bool -StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.BitFieldOverflow.Fail = 2 -> StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.BitFieldOverflow.Saturate = 1 -> StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.BitFieldOverflow.Wrap = 0 -> StackExchange.Redis.BitFieldOverflow -StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.StringBitFieldAsync(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task!>! -StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, StackExchange.Redis.BitFieldOperation operation, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long? -StackExchange.Redis.IDatabase.StringBitField(StackExchange.Redis.RedisKey key, System.ReadOnlyMemory operations, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease! -StackExchange.Redis.RedisFeatures.BitFieldReadOnly.get -> bool +override StackExchange.Redis.ClusterSlotNode.ToString() -> string! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.AzureOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.DefaultOptionsProvider.ToString() -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.AbortOnConnectFail.get -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.IsMatch(System.Net.EndPoint! endpoint) -> bool +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisCloudOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Name.get -> string! +override StackExchange.Redis.Configuration.RedisEnterpriseOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +StackExchange.Redis.ConfigurationOptions.TopologyRefreshSeconds.get -> int +StackExchange.Redis.ConfigurationOptions.TopologyRefreshSeconds.set -> void +StackExchange.Redis.ProductVariant.Dragonfly = 3 -> StackExchange.Redis.ProductVariant +StackExchange.Redis.ProductVariant.Memurai = 4 -> StackExchange.Redis.ProductVariant +StackExchange.Redis.ProductVariant.Redict = 5 -> StackExchange.Redis.ProductVariant +StackExchange.Redis.ProductVariant.KeyDB = 6 -> StackExchange.Redis.ProductVariant static StackExchange.Redis.BitFieldEncoding.Int16.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Int32.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Int64.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Int8.get -> StackExchange.Redis.BitFieldEncoding -static StackExchange.Redis.BitFieldEncoding.operator ==(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool -static StackExchange.Redis.BitFieldEncoding.operator !=(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool static StackExchange.Redis.BitFieldEncoding.Signed(int width) -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt16.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt32.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt63.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.UInt8.get -> StackExchange.Redis.BitFieldEncoding static StackExchange.Redis.BitFieldEncoding.Unsigned(int width) -> StackExchange.Redis.BitFieldEncoding +static StackExchange.Redis.BitFieldEncoding.operator !=(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool +static StackExchange.Redis.BitFieldEncoding.operator ==(StackExchange.Redis.BitFieldEncoding x, StackExchange.Redis.BitFieldEncoding y) -> bool static StackExchange.Redis.BitFieldOffset.Bit(long bit) -> StackExchange.Redis.BitFieldOffset -static StackExchange.Redis.BitFieldOffset.implicit operator StackExchange.Redis.BitFieldOffset(long bit) -> StackExchange.Redis.BitFieldOffset static StackExchange.Redis.BitFieldOffset.Element(long element) -> StackExchange.Redis.BitFieldOffset -static StackExchange.Redis.BitFieldOffset.operator ==(StackExchange.Redis.BitFieldOffset x, StackExchange.Redis.BitFieldOffset y) -> bool +static StackExchange.Redis.BitFieldOffset.implicit operator StackExchange.Redis.BitFieldOffset(long bit) -> StackExchange.Redis.BitFieldOffset static StackExchange.Redis.BitFieldOffset.operator !=(StackExchange.Redis.BitFieldOffset x, StackExchange.Redis.BitFieldOffset y) -> bool +static StackExchange.Redis.BitFieldOffset.operator ==(StackExchange.Redis.BitFieldOffset x, StackExchange.Redis.BitFieldOffset y) -> bool static StackExchange.Redis.BitFieldOperation.Get(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset) -> StackExchange.Redis.BitFieldOperation static StackExchange.Redis.BitFieldOperation.IncrementBy(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset, long value, StackExchange.Redis.BitFieldOverflow overflow = StackExchange.Redis.BitFieldOverflow.Wrap) -> StackExchange.Redis.BitFieldOperation -static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool -static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool static StackExchange.Redis.BitFieldOperation.Set(StackExchange.Redis.BitFieldEncoding encoding, StackExchange.Redis.BitFieldOffset offset, long value, StackExchange.Redis.BitFieldOverflow overflow = StackExchange.Redis.BitFieldOverflow.Wrap) -> StackExchange.Redis.BitFieldOperation -StackExchange.Redis.ProductVariant.Dragonfly = 3 -> StackExchange.Redis.ProductVariant -StackExchange.Redis.ProductVariant.Memurai = 4 -> StackExchange.Redis.ProductVariant -StackExchange.Redis.ProductVariant.Redict = 5 -> StackExchange.Redis.ProductVariant -StackExchange.Redis.ProductVariant.KeyDB = 6 -> StackExchange.Redis.ProductVariant +static StackExchange.Redis.BitFieldOperation.operator !=(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool +static StackExchange.Redis.BitFieldOperation.operator ==(StackExchange.Redis.BitFieldOperation x, StackExchange.Redis.BitFieldOperation y) -> bool +virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.Name.get -> string? +virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.TopologyRefreshInterval.get -> System.TimeSpan diff --git a/src/StackExchange.Redis/RedisLiterals.cs b/src/StackExchange.Redis/RedisLiterals.cs index 12b85ea2f..58d4632c9 100644 --- a/src/StackExchange.Redis/RedisLiterals.cs +++ b/src/StackExchange.Redis/RedisLiterals.cs @@ -72,6 +72,12 @@ public static readonly RedisValue LIMIT = RedisValue.FromRaw("LIMIT"u8), LIST = RedisValue.FromRaw("LIST"u8), LT = RedisValue.FromRaw("LT"u8), + MAINT_NOTIFICATIONS = RedisValue.FromRaw("MAINT_NOTIFICATIONS"u8), + moving_endpoint_type = RedisValue.FromRaw("moving-endpoint-type"u8), + internal_ip = RedisValue.FromRaw("internal-ip"u8), + internal_fqdn = RedisValue.FromRaw("internal-fqdn"u8), + external_ip = RedisValue.FromRaw("external-ip"u8), + external_fqdn = RedisValue.FromRaw("external-fqdn"u8), MATCH = RedisValue.FromRaw("MATCH"u8), MALLOC_STATS = RedisValue.FromRaw("MALLOC-STATS"u8), MAX = RedisValue.FromRaw("MAX"u8), @@ -97,6 +103,8 @@ public static readonly RedisValue PATTERN = RedisValue.FromRaw("PATTERN"u8), PAUSE = RedisValue.FromRaw("PAUSE"u8), PERSIST = RedisValue.FromRaw("PERSIST"u8), + OFF = RedisValue.FromRaw("OFF"u8), + ON = RedisValue.FromRaw("ON"u8), PING = RedisValue.FromRaw("PING"u8), PREPARE = RedisValue.FromRaw("PREPARE"u8), PURGE = RedisValue.FromRaw("PURGE"u8), diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 68662aa29..4b2968ead 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -25,6 +25,7 @@ public static readonly ResultProcessor TrackSubscriptions = new TrackSubscriptionsProcessor(null), Tracer = new TracerProcessor(false), EstablishConnection = new TracerProcessor(true), + MaintenanceNotifications = new MaintenanceNotificationsProcessor(), BackgroundSaveStarted = new ExpectBasicStringProcessor(Literals.background_saving_started.Hash, startsWith: true), BackgroundSaveAOFStarted = new ExpectBasicStringProcessor(Literals.background_aof_rewriting_started.Hash, startsWith: true); @@ -3206,6 +3207,44 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } + /// + /// Handles the reply to the maintenance-notification opt-in, which we send speculatively: a server that + /// doesn't know the subcommand replies with an error, and that is an expected outcome rather than a + /// fault. So the error is absorbed here rather than going through the common error path, which would + /// raise an to the consumer for something we asked for + /// on their behalf. + /// + private sealed class MaintenanceNotificationsProcessor : ResultProcessor + { + public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) + { + reader.MovePastBof(); + var server = connection.BridgeCouldBeNull?.ServerEndPoint; + if (reader.IsError) + { + server?.OnMaintenanceNotificationsRefused(connection, reader.ReadString() ?? "declined"); + SetResult(message, false); + return true; + } + + if (reader.IsScalar && Literals.OK.Hash.IsCS(reader.TryGetSpan(out var span) ? span : reader.Buffer(stackalloc byte[16]))) + { + server?.OnMaintenanceNotificationsAccepted(connection); + SetResult(message, true); + return true; + } + + // anything else: treat as "not available" rather than a protocol fault; being liberal in what + // we accept matters more here than pinning an unverifiable reply shape + server?.OnMaintenanceNotificationsRefused(connection, $"unexpected reply: {reader.GetOverview()}"); + SetResult(message, false); + return true; + } + + protected override bool SetResultCore(PhysicalConnection connection, Message message, ref RespReader reader) + => throw new NotSupportedException(); // SetResult is fully overridden + } + private sealed class TracerProcessor(bool establishConnection) : ResultProcessor { public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs new file mode 100644 index 000000000..99cbffd10 --- /dev/null +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -0,0 +1,820 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite; +using StackExchange.Redis.Maintenance; + +namespace StackExchange.Redis; + +internal sealed partial class ServerEndPoint +{ + private volatile bool _maintenanceNotificationsActive, _maintenanceNotificationsRequested; + private volatile string? _maintenanceNotificationsRefusal; + + /// + /// Whether this server has accepted our request for maintenance notifications on the current connection. + /// + /// + /// Per-connection state, so this is cleared at the start of every handshake and re-established from the + /// reply; the scope is deliberately the server rather than the multiplexer, so one lagging node doesn't + /// disable the feature for the whole deployment. + /// + internal bool MaintenanceNotificationsActive => _maintenanceNotificationsActive; + + [Experimental(Experiments.MaintenanceNotifications, UrlFormat = Experiments.UrlFormat)] + private MaintenanceNotificationMode MaintenanceMode + => Multiplexer.RawConfig.MaintenanceNotifications; + + /// + /// Whether to ask this server for maintenance notifications during handshake. + /// + /// + /// The feature is RESP3-only: the notifications are out-of-band push frames on the connection that + /// carries commands, and a RESP2 connection can have the request accepted and then silently receive + /// nothing - so we don't ask unless we asked for RESP3. (Note that not asking is not the same as being + /// satisfied: under the reconcile below fails the + /// connection for exactly this case.) We can't know what was *negotiated* at write time + /// (the handshake is pipelined, and HELLO hasn't been answered yet), so this tests what we asked + /// for and settles it once the reply has been processed. + /// + private bool ShouldRequestMaintenanceNotifications(bool isInteractive, bool negotiateResp3) + => isInteractive + && negotiateResp3 + && MaintenanceMode != MaintenanceNotificationMode.Disabled + && Multiplexer.CommandMap.IsAvailable(RedisCommand.CLIENT); + + /// + /// The server agreed to send them. + /// + /// + /// Logged as well as recorded, so that the connect log answers "is this actually on?" outright. Previously + /// only the *refusal* was logged, which meant a working feature left no trace and could only be inferred + /// from the absence of a complaint - and that is indistinguishable from never having asked. + /// + /// + /// The wire value for the configured moving-endpoint-type, or null to send no preference. + /// + private RedisValue MaintenanceMovingEndpointTypeLiteral(PhysicalConnection connection) + { + var configured = Multiplexer.RawConfig.MaintenanceMovingEndpointType; + if (configured == MaintenanceEndpointType.Auto) + { + // classify the address we actually reached, not the endpoint we dialled - the latter is usually a + // name, and where it resolved to is what decides whether we are inside the deployment's network + configured = MaintenanceEndpointTypeResolver.Derive( + (connection.VolatileSocket?.RemoteEndPoint as IPEndPoint)?.Address, + connection.IsEncrypted); + } + + return ToLiteral(configured); + } + + private static RedisValue ToLiteral(MaintenanceEndpointType type) => type switch + { + MaintenanceEndpointType.InternalIp => RedisLiterals.internal_ip, + MaintenanceEndpointType.InternalFqdn => RedisLiterals.internal_fqdn, + MaintenanceEndpointType.ExternalIp => RedisLiterals.external_ip, + MaintenanceEndpointType.ExternalFqdn => RedisLiterals.external_fqdn, + MaintenanceEndpointType.None => RedisLiterals.none, + _ => RedisValue.Null, // ServerDefault: a bare ON, which is what we have always sent + }; + + internal void OnMaintenanceNotificationsAccepted(PhysicalConnection connection) + { + _maintenanceNotificationsActive = true; + Multiplexer.Logger?.LogInformationMaintenanceNotificationsAccepted(new(this)); + } + + /// + /// The server declined our request. Recorded rather than acted on: whether that matters is a question for + /// , which sees the negotiated protocol too. + /// + internal void OnMaintenanceNotificationsRefused(PhysicalConnection connection, string reason) + { + _maintenanceNotificationsActive = false; + _maintenanceNotificationsRefusal = reason; + + // via the configured logger, not OnDetailLog: that is [Conditional("PARSE_DETAIL")] and compiles away + // in any normal build, so for as long as it was the only report of a refusal, the reason a server + // declined was invisible to everybody who was not debugging the parser. + Multiplexer.Logger?.LogInformationMaintenanceNotificationsRefused(new(this), reason); + } + + /// + /// Settles the feature for this connection now that the handshake is complete and the protocol is known. + /// + /// + /// The opt-in reply precedes the tracer on the same pipelined connection, so by the time this runs every + /// fact is in: whether we asked, what the server said, and what protocol we ended up on. + /// + private void ReconcileMaintenanceNotifications(PhysicalConnection connection) + { + bool resp3 = connection.Protocol is >= RedisProtocol.Resp3; + if (!resp3) + { + // whatever the server said about the opt-in, nothing can arrive on a RESP2 connection + _maintenanceNotificationsActive = false; + } + + if (_maintenanceNotificationsActive || MaintenanceMode != MaintenanceNotificationMode.Enabled) + { + return; + } + + // Enabled means required: no notifications, no connection. That includes a configuration that never + // got as far as asking - requiring a RESP3-only feature over RESP2 is a contradiction, and failing it + // is more useful than honouring half of it. Note the cross-client spec only calls for failing when + // the *server* errors; extending that to the RESP2 cases is ours, and deliberate + var reason = !resp3 + ? (_maintenanceNotificationsRequested ? "the connection negotiated RESP2" : "RESP3 was not requested") + : _maintenanceNotificationsRefusal ?? "the server did not accept the request"; + + connection.RecordConnectionFailed( + ConnectionFailureType.ProtocolFailure, + new RedisConnectionException(ConnectionFailureType.ProtocolFailure, CommandFlags.None, $"Maintenance notifications are enabled, but unavailable: {reason}", innerException: null)); + } + + // The relaxed-timeout window, expressed as a single deadline in Environment.TickCount terms so that it + // can be read without a lock from the heartbeat sweeps. Zero means "no window"; a deadline that computes + // to zero is nudged by a tick rather than complicating the sentinel. Comparisons are unchecked + // subtraction, as elsewhere in this type, so the ~49-day wrap is a non-event. + private int _relaxedDeadlineTicks; + + // the notification that last touched the window, reported on faults that happen inside it + private int _relaxedType; + + // Dedup state, per notification type. No specification defines the sequence ids, but they were observed on + // Enterprise 8.6.2 to be monotonic per database and shared across types, and to carry the same value on + // every node broadcasting a given event - so they identify the event, which is exactly what dedup needs. + // Keyed per type anyway: within a type the ids are still monotonic, and a per-type key cannot mistake one + // node's earlier event for a replay of another's later one. Allocated on first notification, so a server + // that never sees one pays nothing. + private long[]? _lastSequenceIds; + private bool[]? _haveSequenceIds; // zero is a real sequence number, so "unset" needs its own bit + private readonly object _maintenanceSync = new(); + + /// + /// Whether timeouts are currently relaxed for this server. + /// + internal bool IsMaintenanceRelaxed => GetRelaxedRemaining() > 0; + + /// + /// The notification in force for this server, or if no + /// window is open; reported on faults so that a timeout during a migration says so. + /// + internal MaintenanceNotificationType ActiveMaintenanceType + => GetRelaxedRemaining() > 0 ? (MaintenanceNotificationType)Volatile.Read(ref _relaxedType) : MaintenanceNotificationType.None; + + /// + /// The effective timeout for a command against this server: the configured value, or the relaxed value if + /// that is larger and a window is open. Relaxation is a floor and can never shorten a timeout. + /// + /// + /// Read from the timeout sweeps rather than stamped onto each message: both sweeps rely on head-of-line + /// ordering and stop at the first message that has not timed out, which per-message timeouts would + /// invalidate. The consequence is that relaxation applies to whatever is outstanding when a window opens, + /// and stops applying when it closes - which is one of the reasons the post-event tail exists. + /// + internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) + { + if (GetRelaxedRemaining() <= 0) return configuredMilliseconds; + + var relaxed = (int)Multiplexer.RawConfig.MaintenanceRelaxedTimeout.TotalMilliseconds; + return relaxed > configuredMilliseconds ? relaxed : configuredMilliseconds; + } + + /// + /// Milliseconds of relaxation left, or zero if none; clears an expired window as a side-effect. + /// + private int GetRelaxedRemaining() + { + var deadline = Volatile.Read(ref _relaxedDeadlineTicks); + if (deadline == 0) return 0; + + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining > 0) return remaining; + + // expired; clear it, but only if nobody has moved it on in the meantime + Interlocked.CompareExchange(ref _relaxedDeadlineTicks, 0, deadline); + return 0; + } + + /// + /// An announced disruption has started (or is still running): open or extend the relaxed window. + /// + /// + /// Whether this notification was new. A replay extends nothing and, importantly, must not be *acted* on - + /// see the handoff, where re-acting on a repeat is a feedback loop rather than merely wasted work. + /// + internal bool OnMaintenanceWindowOpened(MaintenanceNotificationType type, long? sequenceId, TimeSpan? time) + { + if (!TryClaimSequenceId(type, sequenceId)) return false; + + var config = Multiplexer.RawConfig; + var floor = config.MaintenanceRelaxedTimeout; + var cap = config.MaintenanceRelaxedWindowMax; + + // no time, or a time at or below zero, means "act now" rather than "ignore this": a connection that + // arrives mid-window is told what is left of it, and that can legitimately be negative + var duration = time is { } value && value > TimeSpan.Zero ? value : floor; + if (duration < floor) duration = floor; + if (duration > cap) duration = cap; + + Volatile.Write(ref _relaxedType, (int)type); + ExtendRelaxedWindow(duration, $"{type} for {duration.TotalSeconds}s"); + return true; + } + + /// + /// An announced disruption has finished: replace the remaining window with the post-event tail. + /// + /// + /// Replace rather than extend-or-shorten. The server has told us the operation completed, so whatever it + /// previously said about duration is stale - but the tail still applies, because completion is when every + /// other client that received the same notification re-engages. + /// + /// Which completion this is. + /// The server's sequence number, for repeat detection. + /// + /// Whether this arrived as part of establishing the connection rather than on a live one - in which case + /// it is the server's retained copy of an event that has already finished, and gets no tail. + /// + internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? sequenceId, bool isCatchUp) + { + if (!TryClaimSequenceId(type, sequenceId)) return; + + // A completion delivered while we were still connecting is the server's catch-up channel, and + // measurement says that channel has no age limit: the same FAILED_OVER was replayed to fresh + // connections three hours after the failover, and completions carry no time field, so nothing in the + // frame distinguishes "just happened" from "happened this morning". Without this, every new + // connection to a database that had ever failed over began life with the full post-event tail of + // relaxed timeouts, and reported any timeout inside it as caused by maintenance that was long over. + // + // Note this declines to *open* a window rather than closing one. Relaxation belongs to the + // ServerEndPoint and is shared by both bridges, so a catch-up arriving on a reconnecting subscription + // bridge must not cancel a window that a live notification opened on the established interactive one. + if (isCatchUp) + { + Multiplexer.Trace($"{type}: catch-up copy of a finished event; no relaxation", ToString()); + return; + } + + Volatile.Write(ref _relaxedType, (int)type); + var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; + if (tail <= TimeSpan.Zero) + { + Volatile.Write(ref _relaxedDeadlineTicks, 0); + Volatile.Write(ref _relaxedEndedTicks, NudgeFromZero(Environment.TickCount)); + Multiplexer.Trace($"{type}: relaxation ended", ToString()); + return; + } + + var deadline = NudgeFromZero(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); + Volatile.Write(ref _relaxedDeadlineTicks, deadline); + Volatile.Write(ref _relaxedEndedTicks, deadline); + Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); + } + + private volatile EndPoint? _handoffTarget; + private int _handoffTargetExpiryTicks; + private int _handoffInFlight, _handoffRecycles; + + /// + /// Where the next connection attempt should go, when a server has named a replacement. + /// + /// + /// Deliberately a *connect* target rather than a new endpoint in the collection. This + /// keeps its identity, its place in server selection, and - the part that + /// matters most - its TLS host: certificate validation and SNI are derived from + /// , so moving the socket without moving the endpoint means a handoff + /// cannot perturb them. Adding the moved-to address as an endpoint would; that is a documented trap in the + /// cross-client contract. + /// + /// Expires, and that is not decoration. Without it, a server that named an address which turns out to be + /// unreachable would pin this endpoint to it for the lifetime of the multiplexer, because every reconnect + /// would keep trying the same dead target. On expiry we fall back to resolving the endpoint normally, which + /// is what we would have done anyway. + /// + /// + internal EndPoint? HandoffTarget + { + get + { + var target = _handoffTarget; + if (target is null) return null; + + if (unchecked(Environment.TickCount - Volatile.Read(ref _handoffTargetExpiryTicks)) >= 0) + { + _handoffTarget = null; // expired; resolve the endpoint the usual way from here on + return null; + } + + return target; + } + } + + /// + /// Points the next connection attempt at a named replacement, for as long as the announced window lasts. + /// + internal void SetHandoffTarget(EndPoint target, TimeSpan window) + { + Volatile.Write(ref _handoffTargetExpiryTicks, unchecked(Environment.TickCount + (int)Math.Max(window.TotalMilliseconds, 1000))); + _handoffTarget = target; + } + + /// + /// Forgets any handoff target, once a connection has been established. + /// + /// + /// Called on full establishment rather than on the connect attempt: if the attempt fails we want the next + /// one to try the target again, within its window. Once a connection is up, normal resolution resumes - + /// by then DNS has usually caught up anyway. + /// + internal void ClearHandoffTarget() => _handoffTarget = null; + private volatile string? _lastHandoffOutcome; + + /// + /// What the last MOVING handoff decided, and why. + /// + /// + /// Recorded rather than only traced because Multiplexer.Trace is [Conditional("VERBOSE")] - it + /// compiles away in any normal build, so a handoff that misbehaves in production would leave no trace at + /// all. This is the minimum that survives: what was decided, readable afterwards. + /// + internal string? LastHandoffOutcome => _lastHandoffOutcome; + + /// How many times a handoff has replaced this server's connections. + internal int HandoffRecycles => Volatile.Read(ref _handoffRecycles); + + /// + /// Acts on a MOVING: find where to go, then get off this connection before we are pushed. + /// + /// + /// The value here is entirely in the *timing*. Without it we already survive a MOVING - the socket + /// closes and we reconnect - but the reconnect happens when the server decides, and re-resolves to whatever + /// DNS says at that moment, which has been measured as still naming the node being retired. Measured on a + /// real deployment: MOVING arrives about six seconds in, DNS moves somewhere between four and + /// nineteen seconds later, and the socket closes seventeen to nineteen seconds after the notification. So + /// the window exists to be *used*, and using it means waiting for DNS to move and then choosing the moment. + /// + /// Fire-and-forget by design: this runs while the connection it concerns is still serving commands, and the + /// notification is delivered on the read loop, which must not wait for a DNS poll. + /// + /// + internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalConnection connection) + { + // One at a time per server. A rolling operation delivers one MOVING per connection, so a second one + // arriving while a handoff is in flight is a repeat or a much later event; either way, starting a + // second poll against the same endpoint achieves nothing. + if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) + { + Multiplexer.Trace("MOVING: a handoff is already in flight", ToString()); + return; + } + + var budget = window is { } value && value > TimeSpan.Zero + ? value + : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor + var current = (connection.VolatileSocket?.RemoteEndPoint as IPEndPoint)?.Address; + + _ = HandoffAsync(budget, successor, current); + } + + private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? currentAddress) + { + var watch = ValueStopwatch.StartNew(); + var replaced = false; + try + { + // Spread the fleet, but only by a fraction of the window - see MaintenanceHandoff.GetJitter for why + // a flat delay is wrong here. + var jitter = MaintenanceHandoff.GetJitter(window, RandomFor(this)); + if (jitter > TimeSpan.Zero) await Task.Delay(jitter).ForAwait(); + + var remaining = window - jitter; + var decision = await MaintenanceHandoff.DecideAsync( + EndPoint, + successor, + currentAddress, + remaining, + pollInterval: TimeSpan.FromSeconds(1), // records carry a 5s TTL, so this is several looks per record + resolve: Multiplexer.AddressResolver, + log: message => Multiplexer.Trace(message, ToString())).ForAwait(); + + Multiplexer.Trace($"MOVING: {decision}", ToString()); + _lastHandoffOutcome = decision.ToString(); + + // Trace is [Conditional("VERBOSE")], so without this a handoff leaves no record in a normal build - + // and a handoff replaces connections, which is exactly the kind of thing somebody needs to be able + // to find afterwards. + Multiplexer.Logger?.LogInformationMaintenanceHandoff(new(this), _lastHandoffOutcome); + switch (decision.Action) + { + case HandoffAction.Recycle: + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + replaced = true; + break; + case HandoffAction.RecycleAtHalfWindow: + // The contract's rule for "no replacement named", applied where there is nothing better to + // go on. Half of the *announced* window, less whatever the jitter already spent. + var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; + if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); + await DrainThenRecycleAsync(window - jitter - (half > TimeSpan.Zero ? half : TimeSpan.Zero), decision.Reason).ForAwait(); + replaced = true; + break; + case HandoffAction.MoveTo when decision.Target is { } target: + // Point the next connection at the named address and replace the connections. Previously + // this only re-read the topology and recycled, which measurably did not work: we recycled + // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were + // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. + SetHandoffTarget(target, remaining); + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + replaced = true; + break; + default: + // Nothing to do is a legitimate outcome, not a failure: the server closes the socket, the + // reconnect re-resolves, and the relaxed window covers the gap. + break; + } + + if (replaced) await WarnIfNotEstablishedInTimeAsync(watch, window).ForAwait(); + } + catch (Exception ex) + { + Multiplexer.OnInternalError(ex, EndPoint); + } + finally + { + Volatile.Write(ref _handoffInFlight, 0); + } + } + + /// + /// Lets in-flight caller work finish, then replaces the connections. + /// + /// + /// Draining first because the socket is still working: anything already written may still be answered, and + /// dropping it would fail commands that were about to succeed. Bounded by what is left of the window, + /// because the server closes the socket at the end of it regardless - so anything not drained by then was + /// going to fail either way, and draining strictly dominates. + /// + /// Both bridges, not just the one that was told. The measured blast radius is the *node*: four connections + /// to one proxy, differing only in handshake, all closed simultaneously, and only the ones that had opted in + /// were warned. + /// + /// + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) + { + var watch = ValueStopwatch.StartNew(); + while (HasCallerWork() && watch.ElapsedMilliseconds < budget.TotalMilliseconds) + { + await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); + } + + var drained = !HasCallerWork(); + var recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); + if (recycled) Interlocked.Increment(ref _handoffRecycles); + Multiplexer.Trace( + $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" + + (drained ? " (drained)" : " (still busy; the window ran out)"), + ToString()); + } + + /// + /// Warns when a replacement connection is not fully established by the time the announced window ends. + /// + /// + /// The contract asks for the new connection to be *fully established* - handshake complete, not merely + /// socket-connected - before the deadline, and to say so when that is exceeded. It is the difference + /// between a handoff that worked and one the server finished for us by closing the socket, and from the + /// outside those look identical: commands succeed either way, because the relaxed window covers the gap. + /// + /// Only reached when connections were actually replaced. Where the decision was to do nothing, waiting for + /// the server to close the socket *is* the plan, so reconnecting after the deadline is the intended path + /// rather than a miss, and warning about it would be noise. + /// + /// + private async Task WarnIfNotEstablishedInTimeAsync(ValueStopwatch watch, TimeSpan window) + { + var deadline = window.TotalMilliseconds; + while (watch.ElapsedMilliseconds < deadline) + { + if (isDisposed) return; // nothing to report about an endpoint that has gone away + if (IsConnected && (IsSubscriberConnected || !SupportsSubscriptions)) return; // in time + await Task.Delay(50).ForAwait(); + } + + if (isDisposed || (IsConnected && (IsSubscriberConnected || !SupportsSubscriptions))) return; + + Multiplexer.Logger?.LogWarningMaintenanceHandoffMissedDeadline( + new(this), + (long)window.TotalMilliseconds, + IsConnected, + IsSubscriberConnected); + } + + [ThreadStatic] + private static Random? _random; + + /// + /// A per-thread , seeded so that two processes handing off at once do not pick the same + /// jitter. + /// + private static Random RandomFor(ServerEndPoint server) => + _random ??= new Random(Environment.TickCount ^ server.GetHashCode()); + + private void ExtendRelaxedWindow(TimeSpan duration, string cause) + { + var candidate = NudgeFromZero(unchecked(Environment.TickCount + (int)duration.TotalMilliseconds)); + while (true) + { + var current = Volatile.Read(ref _relaxedDeadlineTicks); + + // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer + // MIGRATING window must not cut it short + if (current != 0 && unchecked(candidate - current) <= 0) return; + if (Interlocked.CompareExchange(ref _relaxedDeadlineTicks, candidate, current) == current) + { + Volatile.Write(ref _relaxedEndedTicks, candidate); + Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); + return; + } + } + } + + /// + /// When the most recent window was due to end, whether or not it has; zero if there has never been one. + /// + /// + /// A mirror of the deadline rather than a record of when it was cleared, which is what makes it cheap: + /// expiry is lazy (it happens on the next read), so there is no single moment at which to stamp "the + /// window closed", and the deadline already *is* that moment. + /// + private int _relaxedEndedTicks; + + /// How long after a window closes a fault may still be attributed to it, at minimum. + /// + /// The bridge heartbeat raises timeouts on roughly a one-second cadence rather than at the deadline, so a + /// timeout can be reported up to about a second after the moment it actually expired. + /// + private const int FaultAttributionFloorMilliseconds = 1000; + + /// + /// Which notification to blame for a fault, which is a different question from + /// . + /// + /// + /// A command that timed out was outstanding for its whole timeout before anybody noticed, and the + /// heartbeat that notices runs about once a second - so by the time an exception is built, a window that + /// covered the command's entire life may already have closed. Reading the *active* type then reports + /// for a timeout that maintenance plainly caused, which is + /// the opposite of what this property exists for. + /// + /// So a window that closed no longer ago than the command could have been waiting still counts. That is + /// the tightest bound that catches every genuine case, and it is deliberately a bound rather than a + /// certainty: without per-message state - which the timeout sweeps rule out, see + /// - a client cannot know whether *this* command overlapped + /// the window, only whether it could have. + /// + /// + /// The timeout that applied to the faulted command. + internal MaintenanceNotificationType GetMaintenanceTypeForFault(int timeoutMilliseconds) + { + var active = ActiveMaintenanceType; + if (active != MaintenanceNotificationType.None) return active; + + var ended = Volatile.Read(ref _relaxedEndedTicks); + if (ended == 0) return MaintenanceNotificationType.None; // never had a window at all + + var since = unchecked(Environment.TickCount - ended); + var grace = Math.Max(timeoutMilliseconds, FaultAttributionFloorMilliseconds); + return since >= 0 && since <= grace + ? (MaintenanceNotificationType)Volatile.Read(ref _relaxedType) + : MaintenanceNotificationType.None; + } + + /// + /// Zero is the "no window" sentinel, so a deadline that lands on it moves by a tick. + /// + private static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; + + /// + /// Whether this notification is new, per the conservative dedup described on . + /// + private bool TryClaimSequenceId(MaintenanceNotificationType type, long? sequenceId) + { + // no id, no dedup - but the notification still counts. Dropping an announced disruption because we + // could not read a field whose meaning nobody has defined would be the wrong way round + if (sequenceId is not { } id) return true; + + var index = (int)type; + lock (_maintenanceSync) + { + var ids = _lastSequenceIds ??= new long[MaintenanceNotificationTypeCount]; + var have = _haveSequenceIds ??= new bool[MaintenanceNotificationTypeCount]; + if ((uint)index >= (uint)ids.Length) return true; // unknown type: don't dedup what we can't index + + // The "have we seen one" bit is separate because zero is a real sequence number - observed on + // Enterprise 8.6.2 as the first event of a chain (`>4 $6 MOVING :0 :15 _`). Treating a stored zero + // as "unset", which an earlier cut did, quietly disabled dedup for whichever notification happened + // to open the chain. + // + // note "<=", not "<": a replay carries the id we already acted on + if (have[index] && id <= ids[index]) + { + Multiplexer.Trace($"{type}: ignoring replayed sequence id {id}", ToString()); + return false; + } + + ids[index] = id; + have[index] = true; + return true; + } + } + + private const int MaintenanceNotificationTypeCount = 8; // None + the seven notification types + + /// + /// How long to smear a notification-triggered topology refresh over. + /// + /// + /// Not configurable, deliberately. The relaxed-window durations are options because their right value is + /// deployment-specific and we invented them; this is a fixed small smear that exists only so that a fleet + /// which was all told the same thing at the same instant does not all query the same node at the same + /// instant. Nobody needs to tune it, and every option is public surface to keep. Promote it if that turns + /// out to be wrong. + /// + /// Note already declines while one refresh is in + /// flight, so the *local* storm is handled; this is purely about the fleet. + /// + /// + private static readonly int MaintenanceRefreshJitterMilliseconds = 1000; + + // 1 while a jittered refresh is scheduled and has not yet started + private int _refreshPending; + + /// + /// Whether this server is one of the sources in a slot-migration delta - i.e. whether the movement being + /// described is movement away from us. + /// + /// + /// Every node in the cluster reports the same movements, so the sender is not implicitly a source and most + /// notifications describe somebody else. Acting only on our own is what stops a fleet re-reading topology + /// because one shard moved somewhere unrelated - go-redis makes the same choice. + /// + /// Resolution goes through the identity map rather than comparing endpoints directly: a node answers to + /// its address and to its announced hostname, and the delta may name either. + /// + /// + private bool IsSourceOf(IReadOnlyList migrations) + { + foreach (var migration in migrations) + { + if (migration.Source is { } source && ReferenceEquals(Multiplexer.TryResolveServerEndPoint(source), this)) + { + return true; + } + } + + return false; + } + + /// + /// Refreshes the topology after slots have moved away from this server, once the fleet has had a moment to + /// spread out. + /// + /// + /// Deliberately only for the cluster family: MIGRATED and FAILED_OVER arrive in proxied + /// deployments where the client addresses a single endpoint, so there is no topology for a refresh to + /// learn - they get relaxation and nothing else. Endpoints left serving nothing are handled by the + /// existing absence-based pruning, which a refresh feeds; there is deliberately no second, faster + /// retirement path here. + /// + internal void OnSlotsMigratedAway(IReadOnlyList migrations) + { + if (migrations.Count == 0 || !IsSourceOf(migrations)) return; + + // Coalesce *before* the jitter, not after. ReconfigureIfNeeded declines only while a refresh is + // actually in flight, and the jitter spreads a burst of notifications out far enough that each one + // completes before the next begins - so relying on that alone turns ten notifications into ten + // topology passes. Found by the test that counts them. + if (Interlocked.CompareExchange(ref _refreshPending, 1, 0) != 0) + { + Multiplexer.Trace("topology refresh already pending; folding this notification into it", ToString()); + return; + } + + var cause = $"slots migrated from {Format.ToString(EndPoint)}"; + Multiplexer.Trace($"{cause}; refreshing topology after jitter", ToString()); + + // fire-and-forget: we are on the read loop, and the refresh must not block it + _ = RefreshAfterJitterAsync(cause, migrations); + } + + private async Task RefreshAfterJitterAsync(string cause, IReadOnlyList migrations) + { + try + { + await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(MaintenanceRefreshJitterMilliseconds)).ForAwait(); + + // deliberately *after* the delay - see the remarks on ResubscribeStrandedShardedChannels + ResubscribeStrandedShardedChannels(migrations); + + // released before the refresh runs, not after: anything that arrives from here on describes a + // state this pass may not have seen, and deserves its own pass + Volatile.Write(ref _refreshPending, 0); + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, cause); + } + catch (Exception ex) + { + // a refresh we failed to start is a missed optimization, not a fault: the next -MOVED still works + Volatile.Write(ref _refreshPending, 0); + Multiplexer.Trace($"topology refresh after {cause} failed: {ex.Message}", ToString()); + } + } + + /// + /// Re-establishes sharded subscriptions that the slot movement stranded and nothing else recovered. + /// + /// + /// Mostly belt-and-braces: a server that migrates a slot also sends an unsolicited SUNSUBSCRIBE, + /// and already resubscribes on that. This adds two things. + /// It is *pre-emptive* where SMIGRATED arrives first, and it *covers* the case where the + /// unsolicited unsubscribe never arrives or is lost - in which case the only other signal is a message + /// that silently stops being delivered, which nothing detects. + /// + /// It also knows which slots moved, so only the affected channels are touched rather than everything + /// subscribed on this server. + /// + /// + /// Why this runs after the jitter, and only for a subscription connected nowhere. An earlier cut ran + /// it immediately, on the reasoning that a silently-unsubscribed subscriber is a correctness problem while a + /// stale slot map is only a round trip. Measured against a fake that emits the realistic sequence, that + /// produced an extra resubscribe per channel - because the unsolicited unsubscribe had already started one, + /// and mid-flight the subscription still looked like ours. Being a genuine *fallback* means acting only + /// once the other path has had its chance, which costs a stranded subscription up to the jitter in + /// recovery time and costs nothing when it was not needed. + /// + /// + /// Measured against the fake's realistic sequence: four (re)subscribes with notifications off, five with + /// them on. The extra one is this fallback acting on a subscription the unsolicited-unsubscribe path left + /// attached to nothing - i.e. it is the feature working, not duplicate work. One extra attempt per + /// stranded channel, not one per notification. + /// + /// + /// Note it resubscribes via this server, not the migration target, reusing + /// unchanged: the outgoing node is the one we know has + /// the new route, and sending there follows the redirect. The target is named in the notification and + /// could be dialled directly, but it may be a node we have never seen, or named in a form we cannot dial, + /// and the redirect path is the one already proven by the SUNSUBSCRIBE case. + /// + /// + private void ResubscribeStrandedShardedChannels(IReadOnlyList migrations) + { + var subscriptions = Multiplexer.GetSubscriptions(); + if (subscriptions.IsEmpty) return; + + var strategy = Multiplexer.ServerSelectionStrategy; + foreach (var pair in subscriptions) + { + var channel = pair.Key; + + // ordinary pub/sub is not slot-bound, so a slot moving says nothing about it + if (!channel.IsSharded) continue; + + var slot = strategy.HashSlot(channel); + if (slot == ServerSelectionStrategy.NoSlot || !IsInMigratedRange(migrations, slot)) continue; + + // Skip only if it is attached somewhere *else* - that means the other path already moved it. Still + // attached to us is the pre-emptive case (the notification beat the unsubscribe, and the + // subscription is now stale); attached nowhere is the stranded case. Both need acting on, and + // distinguishing them from "already handled" is the whole job of this check. + var subscription = pair.Value; + var current = subscription.GetAnyCurrentServer(); + if (current is not null && !ReferenceEquals(current, this)) + { + Multiplexer.Trace($"slot {slot} moved, and {channel} has already moved with it; leaving it alone", ToString()); + continue; + } + + Multiplexer.Trace($"slot {slot} moved; resubscribing {channel}", ToString()); + Multiplexer.DefaultSubscriber.ResubscribeToServer(subscription, channel, this, cause: "smigrated"); + } + } + + private static bool IsInMigratedRange(IReadOnlyList migrations, int slot) + { + foreach (var migration in migrations) + { + foreach (var range in migration.Slots) + { + if (slot >= range.From && slot <= range.To) return true; + } + } + + return false; + } +} diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 17d37a5c0..e7398108c 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -331,7 +331,22 @@ internal bool IsIdle() => !Multiplexer.ServerSelectionStrategy.OwnsAnySlot(this) && (subscription?.SubscriptionCount ?? 0) == 0 && (interactive?.SubscriptionCount ?? 0) == 0 - && GetOutstandingCount() == 0; + && !HasCallerWork(); + + /// + /// Whether a *caller* is waiting on anything here, which is the only kind of work that should stop us + /// retiring a server. + /// + /// + /// Deliberately not , which counts everything. A node the topology has + /// stopped listing still receives our autoconfigure probes on every pass, and nothing answers them, so + /// they accumulate in its backlog: measured at ~170 per pass, growing without bound. Counting those + /// made the node look busy *because* we were looking for it, so it could never be retired - the + /// precondition defeated itself in exactly the case pruning exists for. Keep-alive traffic has the same + /// property, and is excluded by the same test, since both set the internal-call flag. + /// + internal bool HasCallerWork() + => interactive?.HasCallerWork() == true || subscription?.HasCallerWork() == true; /// /// Work this server still owes an answer on: written-and-awaiting-response, plus anything queued in @@ -360,18 +375,23 @@ internal async Task RetireAsync(string reason, TimeSpan drainTimeout, ILogger? l SetUnselectable(UnselectableFlags.Retiring); log?.LogInformationRetiringServer(new(EndPoint), reason); + // Drain what a *caller* is waiting for, not everything outstanding. Our own probes to a node that + // has gone away will never be answered, so draining on the total means always waiting out the full + // timeout before letting go - measured: a departed node accumulates our autoconfigure traffic + // indefinitely (500+ and climbing), so the drain never once completed early. Callers are who the + // drain exists for; nobody is waiting on our keep-alives. // Stopwatch rather than TickCount64: the latter does not exist on the down-level targets var watch = ValueStopwatch.StartNew(); - int outstanding; - while ((outstanding = GetOutstandingCount()) > 0 && watch.ElapsedMilliseconds < drainTimeout.TotalMilliseconds) + while (HasCallerWork() && watch.ElapsedMilliseconds < drainTimeout.TotalMilliseconds) { await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } - if (outstanding > 0) + if (HasCallerWork()) { - // deliberately reported: an abandoned command is exactly what a caller will be asking about - log?.LogInformationRetiringServerAbandoned(new(EndPoint), outstanding); + // deliberately reported: an abandoned command is exactly what a caller will be asking about. + // The count is the total, since that is what is actually being dropped on the floor + log?.LogInformationRetiringServerAbandoned(new(EndPoint), GetOutstandingCount()); } Dispose(); @@ -914,6 +934,61 @@ static async Task OnEstablishingAsyncAwaited(PhysicalConnection connection, Task return Task.CompletedTask; } + /// + /// How many consecutive failures to connect justify re-reading the topology. + /// + /// + /// Three: enough to rule out a single transient refusal, few enough that recovery is seconds. The + /// number matters less than that there *is* one - the failure this addresses lasted 37 hours. + /// + private const int ConnectFailuresBeforeRefresh = 3; + + private int _lastConnectFailureRefreshTicks; + + /// + /// Called when a connection attempt to this endpoint has failed, with the consecutive failure count. + /// + /// + /// The gap this closes: every existing path that re-reads the topology needs somebody *else* to notice + /// first - a notification, a MOVED from a reachable node, a peer's config broadcast. A client + /// with quiet healthy connections and one endpoint that only ever refuses has nobody to tell it, so it + /// dials the dead address indefinitely. That is not hypothetical: a customer's client did exactly that + /// for 37 hours across a Redis Cloud node replacement. + /// + /// Rate-limited to , deliberately reusing the + /// knob that already means "how often may we re-read configuration" rather than inventing one. The + /// limit is the part that makes this safe: the existing gate exists to stop a stampede - a dead + /// endpoint, times a retry loop, times every client in a fleet, each issuing CLUSTER NODES - and + /// removing the gate without replacing the restraint would trade a stuck client for a thundering herd. + /// + /// + /// It repeats rather than firing once, because one refresh is not guaranteed to help: the topology may + /// not have been updated server-side yet. A permanently dead endpoint therefore prompts a re-read at + /// most once per interval until something changes, which is what makes recovery eventual rather than + /// lucky. + /// + /// + internal void OnRepeatedConnectFailure(int consecutiveFailures) + { + if (consecutiveFailures < ConnectFailuresBeforeRefresh || isDisposed) return; + + // nothing to learn about an endpoint we have already decided to let go of + if ((unselectableReasons & UnselectableFlags.Retiring) != 0) return; + + var interval = Math.Max(Multiplexer.RawConfig.ConfigCheckSeconds, 5) * 1000; + var now = Environment.TickCount; + var last = Volatile.Read(ref _lastConnectFailureRefreshTicks); + if (last != 0 && unchecked(now - last) < interval) return; + + if (Interlocked.CompareExchange(ref _lastConnectFailureRefreshTicks, NudgeFromZeroTicks(now), last) != last) return; + + Multiplexer.Logger?.LogInformationRefreshingAfterConnectFailures(new(this), consecutiveFailures); + Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, $"{consecutiveFailures} consecutive connect failures"); + } + + /// Zero means "never", so a tick count that lands on it moves by one. + private static int NudgeFromZeroTicks(int ticks) => ticks == 0 ? 1 : ticks; + internal void OnFullyEstablished(PhysicalConnection connection, string source) { try @@ -924,8 +999,16 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // Clear the unselectable flag ASAP since we are open for business ClearUnselectable(UnselectableFlags.DidNotRespond); + // whatever a handoff pointed us at, we are connected now: resume normal resolution + ClearHandoffTarget(); + // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; + + if (connection is not null && bridge == interactive) + { + ReconcileMaintenanceNotifications(connection); + } if (bridge == subscription || isResp3) { // Note: this MUST be fire and forget, because we might be in the middle of a Sync processing @@ -1241,6 +1324,10 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) // forget what the previous connection's HELLO told us; re-established below, if this one repeats it // (the subscription handshake is deliberately left out of this: it doesn't do the discovery step) RoleKnownFromHello = false; + + // likewise per-connection: re-armed from this handshake's reply, if we ask + _maintenanceNotificationsActive = _maintenanceNotificationsRequested = false; + _maintenanceNotificationsRefusal = null; } // HELLO serves two purposes: negotiating RESP3, and reporting details we would otherwise need INFO or @@ -1332,6 +1419,25 @@ private async Task HandshakeAsync(PhysicalConnection connection, ILogger? log) msg = Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.ID); msg.SetInternalCall(); await WriteDirectOrQueueFireAndForgetAsync(connection, msg, autoConfig ??= ResultProcessor.AutoConfigureProcessor.Create(log)).ForAwait(); + + if (ShouldRequestMaintenanceNotifications(isInteractive, negotiateResp3)) + { + _maintenanceNotificationsRequested = true; + // speculative in the same way as the AUTH above: we don't yet know what HELLO negotiated, + // so we ask whenever we asked for RESP3, and ReconcileMaintenanceNotifications sorts out a + // downgrade once the reply has been processed. A bare ON is explicitly valid: the server + // then picks the endpoint type, which is what we want until we derive one ourselves. + log?.LogInformationRequestingMaintenanceNotifications(new(this), MaintenanceMode); + + // A bare ON leaves the endpoint type to the server, and every MOVING observed that way + // carried no address at all - so when a caller asks for a specific form, say so. + var endpointType = MaintenanceMovingEndpointTypeLiteral(connection); + msg = endpointType.IsNull + ? Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON) + : Message.Create(-1, CommandFlags.FireAndForget | Message.NoFlushFlag, RedisCommand.CLIENT, [RedisLiterals.MAINT_NOTIFICATIONS, RedisLiterals.ON, RedisLiterals.moving_endpoint_type, endpointType]); + msg.SetInternalCall(); + await WriteDirectOrQueueFireAndForgetAsync(connection, msg, ResultProcessor.MaintenanceNotifications).ForAwait(); + } } var bridge = connection.BridgeCouldBeNull; diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 587af0622..f808312da 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -42,6 +42,7 @@ + @@ -59,6 +60,9 @@ + + + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs new file mode 100644 index 000000000..40188bd5b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs @@ -0,0 +1,11 @@ +using Xunit; + +// One cluster, one injector, and scenarios that mutate *cluster* state - node exclusions, maintenance mode, +// endpoint policies. Running classes in parallel therefore breaks two ways at once: they interfere semantically +// (one scenario's teardown restores nodes another is relying on, giving errors like "Need at least 2 nodes with +// shards"), and they starve each other, because the injector processes actions through a queue. Measured: the +// first whole-suite run failed 20 of 26, almost all of them waiting on a queued setup. +// +// So this tier is strictly serial. It costs wall-clock - the scenarios are minutes each - and buys results that +// mean something. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs new file mode 100644 index 000000000..b904632e2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterFamilyScenarioTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The OSS cluster notification family against a real cluster: SMIGRATING, SMIGRATED, and what +/// the client is supposed to do about them. +/// +/// +/// Every /slot-migrate trigger requires oss_cluster: true with all-master-shards, so this +/// scenario family is the cluster half of the feature - the half whose reactions (re-reading the slot map, +/// re-subscribing stranded sharded channels) have until now only been exercised against the in-process fake. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "cluster-family")] +public class ClusterFamilyScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + /// + /// Slot migrations, in the four shapes the injector can produce. + /// + /// + /// The effects differ in what happens to the *endpoint list*, which is the thing a client has to keep up + /// with: remove-add retires a node and introduces one, remove only retires, add only + /// introduces, and slot-shuffle moves slots between nodes that both stay. So they cover the four + /// ways a slot map can go stale, and only one of them (shuffle) leaves the endpoint set alone. + /// + /// + /// Only remove here. The other effects need a database the setup leg will not provision, and get + /// their own fixture below rather than skipping: add and slot-shuffle need a node holding + /// several shards, and remove-add is not in /slot-migrate/setup's effect enum at all. + /// + [Theory] + [InlineData("remove", "migrate")] + public async Task SlotMigrationIsAnnouncedAndActedOn(string effect, string trigger) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "slot-migrate", effect, trigger, log.WriteLine, + setupTrigger: "reshard", cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + ScenarioSupport.RequireEffectIsAchievable(scenario, effect); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + foreach (var migration in push.SlotMigrations) + { + log.WriteLine($" slots {migration.RawSlots}: {migration.Source} -> {migration.Target}"); + } + } + }; + + var endpointsBefore = conn.GetEndPoints().Length; + log.WriteLine($"connected as {conn.GetServer(conn.GetEndPoints()[0]).ServerType} across {endpointsBefore} endpoint(s)"); + + // keys spread across slots, so a migration of any shard moves something we are actually using + var db = conn.GetDatabase(); + var keys = Enumerable.Range(0, 32).Select(i => (RedisKey)$"fi-{effect}-{i}").ToArray(); + foreach (var key in keys) await db.StringSetAsync(key, "before"); + + clock.Restart(); + await ScenarioSupport.FireOrSkipAsync(scenario, effect, cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports finished"); + + // Keep using the connection while things move: a slot map that has gone stale shows up as MOVED + // redirections, and the point of the feature is that we learn the new one rather than eating them. + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + int reads = 0, failures = 0; + while (clock.Elapsed < deadline) + { + foreach (var key in keys) + { + try + { + await db.StringGetAsync(key); + reads++; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + failures++; + if (failures <= 3) log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s read failed: {ex.Message}"); + } + } + + await Task.Delay(500, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s); {reads} reads, {failures} failures; " + + $"endpoints {endpointsBefore} -> {conn.GetEndPoints().Length}"); + + // A slot migration is announced to every proxy, so any connection should see it. What is asserted is + // only that we were told and understood it - the *reaction* is asserted separately below, because a + // migration that moves no slot we hold would legitimately change nothing here. + Assert.NotEmpty(events); + Assert.All(events, e => Assert.NotEqual(MaintenanceNotificationType.None, e.NotificationType)); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.SlotMigrating or MaintenanceNotificationType.SlotMigrated); + + // Any SMIGRATED that carried a payload must have parsed into something usable; an empty list would + // mean we saw the frame and threw the contents away. + foreach (var migrated in events.Where(e => e.NotificationType == MaintenanceNotificationType.SlotMigrated)) + { + if (!string.IsNullOrEmpty(migrated.Payload) || migrated.SlotMigrations.Count > 0) + { + Assert.NotEmpty(migrated.SlotMigrations); + Assert.All(migrated.SlotMigrations, m => Assert.NotNull(m.Source)); + } + } + } + + // and the client is still serving keys across the new topology + Assert.True(reads > 0, "no reads succeeded at all during the migration"); + foreach (var key in keys) Assert.Equal("before", await db.StringGetAsync(key)); + } + + [Fact] + public async Task ShardedSubscriptionSurvivesASlotMigration() + { + // D5's resubscription, against a real cluster. A sharded channel is bound to the node owning its slot, + // so a migration strands the subscription: the server sends an unsolicited SUNSUBSCRIBE and stops + // delivering. Recovering that is the client's job, and until now only the fake has tested it. + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "slot-migrate", "remove", "migrate", log.WriteLine, + setupTrigger: "reshard", cancellationToken: cancellationToken); + Assert.NotNull(scenario.Database); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(scenario.Database.GetClientConfig(fixture.Environment)); + var subscriber = conn.GetSubscriber(); + + // several channels, because the migration moves the slots of one node: with one channel the test would + // usually be asserting that nothing happened to it + var channels = Enumerable.Range(0, 16).Select(i => RedisChannel.Sharded($"fi-shard-{i}")).ToArray(); + var received = new int[channels.Length]; + for (int i = 0; i < channels.Length; i++) + { + var index = i; + await subscriber.SubscribeAsync(channels[i], (_, _) => Interlocked.Increment(ref received[index])); + } + + Assert.True(await DeliversEverywhereAsync(subscriber, channels, received, cancellationToken), + "every sharded channel should deliver before the migration, or the test proves nothing"); + log.WriteLine($"all {channels.Length} sharded channels delivering before the migration"); + + await ScenarioSupport.FireOrSkipAsync(scenario, "remove", cancellationToken); + + // The recovery is deliberately allowed to be slow: it is driven by a jittered topology refresh and, as a + // fallback, by a delayed resubscription sweep. What matters is that it happens without the caller doing + // anything, not that it is instant. + var recovered = await DeliversEverywhereAsync(subscriber, channels, received, cancellationToken, timeoutSeconds: 90); + log.WriteLine($"delivery after migration: {(recovered ? "all channels" : "INCOMPLETE")}"); + Assert.True(recovered, "sharded subscriptions should deliver again once the topology settles"); + } + + /// + /// Publishes to every channel and waits until each one has delivered at least one more message. + /// + /// + /// Publishing repeatedly rather than once: the publish itself routes by slot, so during a migration an + /// individual publish can land on a node that no longer owns the slot. Retrying is what a real caller would + /// do, and it keeps the test measuring *delivery* rather than the timing of one message. + /// + private async Task DeliversEverywhereAsync( + ISubscriber subscriber, + RedisChannel[] channels, + int[] received, + CancellationToken cancellationToken, + int timeoutSeconds = 20) + { + var baseline = channels.Select((_, i) => Volatile.Read(ref received[i])).ToArray(); + var deadline = DateTime.UtcNow.AddSeconds(timeoutSeconds); + + while (DateTime.UtcNow < deadline) + { + for (int i = 0; i < channels.Length; i++) + { + if (Volatile.Read(ref received[i]) > baseline[i]) continue; + try + { + await subscriber.PublishAsync(channels[i], "ping"); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // expected mid-migration; the retry is the point + } + } + + await Task.Delay(500, cancellationToken); + if (channels.Select((_, i) => Volatile.Read(ref received[i]) > baseline[i]).All(x => x)) return true; + } + + var missing = channels.Where((_, i) => Volatile.Read(ref received[i]) <= baseline[i]).Select(c => c.ToString()); + log.WriteLine($" not delivering: {string.Join(", ", missing)}"); + return false; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs new file mode 100644 index 000000000..fafc0c8a6 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ClusterWideFailureScenarioTests.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The whole cluster goes away, and does not come back. +/// +/// +/// Gated separately from the other destructive scenarios, and deliberately so: killing one node damages one +/// node, while this takes out every database on the cluster including anybody else's. It is the last thing you +/// run against a deployment, and **it ends the deployment** - measured 2026-09-03: cluster_failure +/// takes a node_ids list, stops those nodes, and restores nothing. With every node in the list the +/// cluster stays down; rladmin stops answering, and the environment needs re-provisioning. +/// +/// So recovery is not assertable here, and the first version of this test was wrong to try: it asserted the +/// client would come back, which it cannot do when there is nothing to come back to. What *is* assertable is +/// that a total outage degrades cleanly - one reported failure, then commands that fail as Redis exceptions +/// rather than hanging, crashing, or throwing something unrelated - and that the multiplexer stays in a state +/// where it would recover if the deployment did. +/// +/// +/// reset_cluster is deliberately *not* here. It rebuilds the cluster from scratch, so the databases a +/// client was using cease to exist and there is no recovery to observe: it is a lifecycle operation for +/// whoever owns the environment, not a client test, and writing it as one would only produce a test that +/// asserts nothing. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "destructive")] +public class ClusterWideFailureScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string EnableVariable = "SER_FI_CLUSTER_FAILURE"; + + [Fact] + public async Task ATotalOutageFailsCleanlyRatherThanWedging() + { + if (!string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase)) + { + Assert.Skip($"set {EnableVariable}=true to run this; it takes out every database on the cluster, not just ours"); + } + + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + log.WriteLine($"cluster_failure against {database}"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + const string Key = "fi-cluster-failure"; + await db.StringSetAsync(Key, "before"); + + var drops = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (drops) drops.Add(e.FailureType); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + + // Every node, together: the action is "fail these nodes", not "fail the cluster" - it wants a + // `node_ids` list, which the schema does not say and which an exception was kind enough to tell us. + var nodes = await ClusterNodes.ListAsync(fixture.Injector, database.BdbId, cancellationToken); + log.WriteLine($"cluster nodes: {string.Join(", ", nodes.Select(n => $"{n.Id}={n.Role}@{n.ExternalAddress}"))}"); + if (nodes.Count == 0) Assert.Skip("no nodes could be listed, so there is nothing to fail"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "cluster_failure", + new Dictionary + { + ["bdb_id"] = database.BdbId.ToString(), + ["node_ids"] = nodes.Select(n => n.Id).ToArray(), + }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not run 'cluster_failure': {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports cluster_failure finished"); + + // Probe for a while and record *how* it fails. Recovery is not expected - see the remarks - so the + // interesting question is whether every failure is a Redis-family one, which is what a caller can + // write a catch block for. + var faults = new List(); + var recovered = false; + var deadline = clock.Elapsed + TimeSpan.FromSeconds(120); + while (clock.Elapsed < deadline && !recovered) + { + try + { + recovered = db.StringGet(Key) == "before"; + } + catch (Exception ex) + { + lock (faults) faults.Add(ex.GetType().Name); + } + + await Task.Delay(2000, cancellationToken); + } + + lock (drops) + { + log.WriteLine( + $" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered={recovered} after {drops.Count} drop(s): " + + (drops.Count == 0 ? "(none)" : string.Join(", ", drops.Distinct()))); + } + + string[] observedFaults; + lock (faults) observedFaults = [.. faults.Distinct()]; + log.WriteLine($" faults: {(observedFaults.Length == 0 ? "(none)" : string.Join(", ", observedFaults))}"); + + if (recovered) + { + // If the deployment does come back - a smaller node list, or somebody restarting it - then coming + // back with it is the requirement, so say so rather than passing silently. + Assert.Equal("before", await db.StringGetAsync(Key)); + return; + } + + lock (drops) Assert.NotEmpty(drops); // the outage has to have been observed, or this proves nothing + Assert.NotEmpty(observedFaults); + Assert.All(observedFaults, name => Assert.Contains(name, new[] + { + nameof(RedisConnectionException), + nameof(RedisTimeoutException), + nameof(RedisServerException), + nameof(TimeoutException), + })); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs new file mode 100644 index 000000000..2146492a5 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/DenseClusterScenarioTests.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// An OSS cluster with packed shards, so add and slot-shuffle have something to move. +/// +public sealed class DenseClusterFixture() : FaultInjectorFixture(DatabaseShape.OssClusterDense); + +/// +/// The slot migrations the scenario setup legs cannot provision for. +/// +/// +/// These looked like an environment limitation - the injector refuses them with "No node with multiple shards +/// found" - and they are not: the setup leg provisions one shard per node, so there is never a node to take a +/// shard *from*. Six shards with dense placement over three nodes gives two per node, and both effects +/// then run. The lesson generalises: a scenario that setup cannot arrange is still reachable by provisioning +/// the database ourselves and driving the run leg by bdb_id. +/// +/// No scenario teardown here, deliberately. Teardown deletes the database, and this database belongs to the +/// fixture, which deletes it itself; and migrate excludes no nodes, so there is nothing to restore. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "cluster-family")] +public class DenseClusterScenarioTests(DenseClusterFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("add")] + [InlineData("slot-shuffle")] + [InlineData("remove-add")] + public async Task SlotMigrationIsAnnouncedAndActedOn(string effect) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + log.WriteLine($"{effect} against {database}"); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + foreach (var migration in push.SlotMigrations) + { + log.WriteLine($" slots {migration.RawSlots}: {migration.Source} -> {migration.Target}"); + } + } + }; + + var db = conn.GetDatabase(); + var keys = Enumerable.Range(0, 32).Select(i => (RedisKey)$"fi-dense-{effect}-{i}").ToArray(); + foreach (var key in keys) await db.StringSetAsync(key, "before"); + + clock.Restart(); + var query = new Dictionary + { + ["effect"] = effect, + ["trigger"] = "migrate", + ["bdb_id"] = database.BdbId.ToString(), + }; + await fixture.Injector.PostScenarioAsync("slot-migrate", leg: null, query, cancellationToken: cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports finished"); + + int reads = 0, failures = 0; + var deadline = clock.Elapsed + TimeSpan.FromSeconds(20); + while (clock.Elapsed < deadline) + { + foreach (var key in keys) + { + try + { + await db.StringGetAsync(key); + reads++; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + failures++; + } + } + + await Task.Delay(500, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s); {reads} reads, {failures} failures"); + Assert.NotEmpty(events); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.SlotMigrating + or MaintenanceNotificationType.SlotMigrated); + } + + Assert.True(reads > 0, "no reads succeeded during the migration"); + foreach (var key in keys) Assert.Equal("before", await db.StringGetAsync(key)); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs new file mode 100644 index 000000000..9b39e2393 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/DestructiveScenarioTests.cs @@ -0,0 +1,247 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The scenarios that break things rather than move them: a shard dies, a node dies, a proxy dies. +/// +/// +/// Held back from every unattended run until now, deliberately - these damage the cluster, and leaving a +/// broken environment behind costs more than the coverage is worth when nobody is watching. Run them at the +/// end of a cluster's life, supervised, which is what this is. +/// +/// The client has nothing feature-specific to do here: a shard or node failing is not announced, so there is no +/// notification to act on and no handoff to perform. That is exactly why they are worth running. Everything +/// this feature adds - relaxed windows, handoffs, endpoint retirement - sits on top of ordinary reconnect and +/// topology handling, so a *silent* failure is the control: if recovery from an unannounced death regressed, +/// the announced paths are resting on sand. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "destructive")] +public class DestructiveScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string EnableVariable = "SER_FI_DESTRUCTIVE"; + + /// + /// Opt-in beyond the tier's own gate, because these cannot be undone. + /// + /// + /// E2E_SCENARIO_TESTS says "you may create and delete databases"; it does not say "you may kill + /// nodes". A cluster that has to be re-provisioned is 10-15 minutes of somebody's afternoon, so the second + /// gate is the difference between a deliberate session and an expensive surprise. + /// + /// + /// Which node node_failure kills; overridable, because which node matters and only the operator knows. + /// + /// + /// Not node 1: that is where the cluster's own management sits on a default install, so killing it takes + /// the fault injector's access with it and the test cannot observe its own outcome. + /// + private static int NodeToKill => + int.TryParse(Environment.GetEnvironmentVariable("SER_FI_NODE_TO_KILL"), out var node) && node > 0 ? node : 2; + + private static bool Enabled => + string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase); + + /// + /// Kills the node that actually serves this database, rather than an arbitrary one. + /// + /// + /// The version of the test below that means something. Measured 2026-09-03: node_failure against a + /// node we were not connected through produced zero connection drops - the deployment absorbed it and the + /// client never noticed - so a green run proved nothing about our recovery. Resolving the endpoint to a + /// node first is what makes the failure land where the client can see it. + /// + [Fact] + public async Task KillingTheNodeThatServesUsIsSurvived() + { + if (!Enabled) Assert.Skip($"set {EnableVariable}=true to run the destructive scenarios; they damage the cluster"); + + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + + var nodes = await ClusterNodes.ListAsync(fixture.Injector, database.BdbId, cancellationToken); + log.WriteLine($"cluster nodes: {string.Join(", ", nodes.Select(n => $"{n.Id}={n.Role}@{n.ExternalAddress}"))}"); + + var serving = await ClusterNodes.FindServingAsync(fixture.Injector, database.BdbId, database.Host, cancellationToken); + if (serving is null) Assert.Skip($"could not map {database.Host} to a node, so this would kill an arbitrary one"); + + log.WriteLine($"{database} is served by node {serving.Id} ({serving.Role}@{serving.ExternalAddress})"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + const string Key = "fi-node-failure-targeted"; + await db.StringSetAsync(Key, "before"); + + var drops = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (drops) drops.Add(e.FailureType); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId}"); + } + }; + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "node_failure", + new Dictionary { ["node_id"] = serving.Id.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not run 'node_failure' against node {serving.Id}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports node_failure finished"); + + var recovered = await Poll.UntilAsync( + () => + { + try + { + return db.StringGet(Key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 180_000, + pollMilliseconds: 1000); + + lock (drops) + { + log.WriteLine( + $" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered={recovered} after {drops.Count} drop(s): " + + (drops.Count == 0 ? "(none)" : string.Join(", ", drops.Distinct()))); + } + + Assert.True(recovered, "the client should recover on its own after the node serving it is killed"); + Assert.Equal("before", await db.StringGetAsync(Key)); + } + + [Theory] + [InlineData("shard_failure", "bdb_id")] + [InlineData("proxy_failure", "bdb_id")] + [InlineData("node_failure", "node_id")] // measured: this one is scoped to a *node*, and rejects bdb_id + public async Task AnUnannouncedFailureIsSurvived(string action, string scope) + { + if (!Enabled) Assert.Skip($"set {EnableVariable}=true to run the destructive scenarios; they damage the cluster"); + + // Provisioned rather than a template database: a shard dying wants replication behind it, and the + // environment's own databases are created without it (and this cluster has none at all). + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + log.WriteLine($"{action} against {database}"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + var key = $"fi-{action}"; + await db.StringSetAsync(key, "before"); + + var drops = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (drops) drops.Add(e.FailureType); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId}"); + } + }; + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + action, + new Dictionary + { + // The scope differs by action and the schema does not say so: shard_failure and + // proxy_failure take the database, node_failure takes a node - discovered by being told + // "Invalid parameter 'node_id': got None, expected valid node ID". + [scope] = scope == "bdb_id" ? database.BdbId.ToString() : NodeToKill.ToString(), + }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + // The parameters for this family are untyped in the injector's schema, so a rejection is a harness + // finding rather than a client one - and recording the message is the point, since it is the only + // documentation of what these actions want. + Assert.Skip($"the injector would not run '{action}' with {scope}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports {action} finished"); + + // Recovery is polled rather than timed: what matters is that the client gets there on its own, and how + // long a real cluster takes to bring a shard back is not ours to assert. + var recovered = await Poll.UntilAsync( + () => + { + try + { + return db.StringGet(key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 120_000, + pollMilliseconds: 1000); + + lock (drops) + { + log.WriteLine( + $" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered={recovered} after {drops.Count} drop(s): " + + (drops.Count == 0 ? "(none)" : string.Join(", ", drops.Distinct()))); + + // Read the drop count before reading anything into a pass. Measured 2026-09-03: only + // proxy_failure was visible to the client at all (one SocketClosed, restored ~8s later); + // shard_failure on a replicated database and node_failure against a node we were not connected + // through both produced *zero* drops. A run with no drops has proved that the deployment absorbed + // the failure, which is worth knowing - but it has not exercised our recovery path, so do not + // count it as coverage of one. Making node_failure bite needs the node that actually serves this + // database, which means resolving the endpoint's address and matching it against the cluster's + // node list; SER_FI_NODE_TO_KILL is the manual version of that. + if (drops.Count == 0) + { + log.WriteLine(" note: the client never lost a connection, so this run tested the deployment rather than the client"); + } + } + + Assert.True(recovered, $"the client should recover on its own from {action} without being told"); + + // and the data survived, which is the deployment's promise rather than ours - stated because a + // "recovery" that silently lost the key would otherwise pass + Assert.Equal("before", await db.StringGetAsync(key)); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs new file mode 100644 index 000000000..de33a1d44 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/CertificateSanity.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Security.Cryptography.X509Certificates; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Checks that the environment's TLS material belongs to the cluster we are about to talk to. +/// +/// +/// Environments get re-provisioned, and the certificates do not always follow. Observed 2026-09-01: the folder +/// held a server certificate for *.marcgravell-test-46be1d08... while the live cluster was +/// marcgravell-test-e21cd75d... - a leftover from a previous provision, three days older than the +/// env_output.json beside it. +/// +/// Without this check, that presents as a TLS handshake failure, which reads like a client bug and is not one: +/// the certificate genuinely does not cover the name being dialled, and refusing it is correct +/// (TrustIssuer tolerates chain errors only, so a name mismatch fails - as it should). Detecting it here +/// turns half an hour of certificate archaeology into a skip that names both clusters, and it costs nothing +/// because it happens before a database is provisioned. +/// +/// +internal static class CertificateSanity +{ + /// + /// Skips when the environment's certificates were issued for a different cluster. + /// + public static void RequireCertificatesMatchThisCluster(FaultInjectorEnvironment environment, Action log) + { + var clusterName = environment.Cluster?.ClusterName; + if (clusterName is null) return; // nothing to compare against; let the connect speak for itself + + // the server certificate the environment generated, if it left one behind + var leafPath = Path.Combine(environment.ConfigDirectory.FullName, "redis.crt"); + if (!File.Exists(leafPath)) return; + + var leaf = X509CertificateLoader.LoadCertificateFromFile(leafPath); + var names = ReadDnsNames(leaf); + log($"environment server certificate covers [{string.Join(", ", names)}]; cluster is '{clusterName}'"); + + if (!names.Any(name => CoversClusterEndpoints(name, clusterName))) + { + Assert.Skip( + $"the environment's TLS certificates cover [{string.Join(", ", names)}] but this cluster is " + + $"'{clusterName}' - they are left over from an earlier provision, so a TLS test here would " + + "only be measuring the mismatch. Re-provision with certificate generation enabled to run it."); + } + } + + /// + /// Every DNS name a certificate carries, not just the first. + /// + /// + /// GetNameInfo returns one name, which is not enough: these certificates carry both a wildcard and + /// the bare cluster name, and which one comes back decides the answer. Reading the SAN extension properly is + /// the difference between a check that works and one that skips a perfectly good environment - as the first + /// version of this did. + /// + private static List ReadDnsNames(X509Certificate2 certificate) + { + foreach (var extension in certificate.Extensions) + { + if (extension is X509SubjectAlternativeNameExtension san) + { + return [.. san.EnumerateDnsNames()]; + } + } + + var subject = certificate.GetNameInfo(X509NameType.DnsName, forIssuer: false); + return subject is null ? [] : [subject]; + } + + /// + /// Whether a certificate name covers the endpoints of a given cluster. + /// + /// + /// The hosts actually dialled are *database* endpoints - redis-13500.<cluster> - so a wildcard + /// whose parent is the cluster name is exactly right, even though (correctly) it does not match the bare + /// cluster name itself. An exact SAN for the cluster name also counts, since that is what the REST API is + /// reached by. + /// + private static bool CoversClusterEndpoints(string certificateName, string clusterName) + { + if (certificateName.StartsWith("*.", StringComparison.Ordinal)) + { + return string.Equals(certificateName[2..], clusterName, StringComparison.OrdinalIgnoreCase); + } + + return string.Equals(certificateName, clusterName, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs new file mode 100644 index 000000000..6e81e1843 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterNodes.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The cluster's node list, read through the fault injector rather than the cluster's own API. +/// +/// +/// The management API on port 9443 is not reachable from outside the deployment's network - measured: a socket +/// error, not an authentication one - so anything a test needs to know about nodes has to come back the same +/// way it gives instructions. execute_rladmin_command runs cluster-side and returns stdout, which makes +/// status nodes the portable source of truth. +/// +/// Text parsing is not lovely, but it is honest about where the information comes from, and the alternative - +/// hardcoding node ids - is what made the first destructive run prove nothing. +/// +/// +internal static class ClusterNodes +{ + internal sealed record Node(int Id, string Role, string Address, string ExternalAddress); + + /// + /// Runs rladmin status nodes and returns what it said. + /// + /// + /// is required by the action even though the command is cluster-wide: the + /// injector resolves a database to decide where to run. + /// + public static async Task> ListAsync(FaultInjectorClient injector, int bdbId, CancellationToken cancellationToken) + { + var result = await injector.RunActionAsync( + "execute_rladmin_command", + new Dictionary + { + ["bdb_id"] = bdbId.ToString(), + ["rladmin_command"] = "status nodes", + }, + cancellationToken: cancellationToken).ConfigureAwait(false); + + // the action's payload nests the command's stdout under output.output + var text = result.ValueKind == JsonValueKind.Object + && result.TryGetProperty("output", out var inner) + && inner.ValueKind == JsonValueKind.Object + && inner.TryGetProperty("output", out var stdout) + ? stdout.GetString() + : null; + + return text is null ? [] : Parse(text); + } + + /// + /// Which node currently answers for this hostname, or null if the addresses do not match any node. + /// + /// + /// The step that makes a node-scoped fault mean anything: killing an arbitrary node usually proves + /// nothing, because the deployment absorbs it and the client never notices. + /// + public static async Task FindServingAsync( + FaultInjectorClient injector, + int bdbId, + string host, + CancellationToken cancellationToken) + { + var nodes = await ListAsync(injector, bdbId, cancellationToken).ConfigureAwait(false); + var addresses = (await Dns.GetHostAddressesAsync(host, cancellationToken).ConfigureAwait(false)) + .Select(a => a.ToString()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + return nodes.FirstOrDefault(n => + addresses.Contains(n.ExternalAddress) || addresses.Contains(n.Address)); + } + + /// + /// Reads the fixed-column output of status nodes. + /// + /// + /// The leading * marks the node the command ran against, so it is stripped rather than parsed. + /// + internal static List Parse(string output) + { + var nodes = new List(); + foreach (var line in output.Split('\n')) + { + var match = Regex.Match( + line.Trim(), + @"^\*?node:(?\d+)\s+(?\S+)\s+(?\S+)\s+(?\S+)"); + if (match.Success && int.TryParse(match.Groups["id"].Value, out var id)) + { + nodes.Add(new Node(id, match.Groups["role"].Value, match.Groups["addr"].Value, match.Groups["ext"].Value)); + } + } + + return nodes; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs new file mode 100644 index 000000000..8cddd2841 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The cluster's own REST API on port 9443, for the few facts the fault injector does not expose. +/// +/// +/// Reads only. Anything that *changes* state goes through the injector so it is recorded as a job with an +/// action id - which is also how the console works, and it means a scenario can be reconstructed afterwards +/// from the injector's history rather than from somebody's memory. +/// +public sealed class ClusterRestClient : IDisposable +{ + private readonly HttpClient _http; + + public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, string? certificateAuthorityPath) + { + var handler = new HttpClientHandler(); + + if (certificateAuthorityPath is not null) + { + // The cluster's management certificate is self-signed per environment, like the proxy ones. Pin to + // the CA in the config directory rather than accepting anything: this channel carries credentials. + var issuer = X509CertificateLoader.LoadCertificateFromFile(certificateAuthorityPath); + handler.ServerCertificateCustomValidationCallback = (_, certificate, chain, _) => + { + if (certificate is null || chain is null) return false; + chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.Add(issuer); + chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + return chain.Build(certificate); + }; + } + + _http = new HttpClient(handler) { BaseAddress = credentials.RestUrl, Timeout = TimeSpan.FromSeconds(30) }; + var basic = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credentials.Username}:{credentials.Password}")); + _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basic); + } + + public void Dispose() => _http.Dispose(); + + /// + /// Every database on the cluster, as (bdb_id, name). + /// + public async Task> ListDatabasesAsync() + { + using var response = await _http.GetAsync("/v1/bdbs?fields=uid,name"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var results = new List<(int, string)>(); + foreach (var bdb in document.RootElement.EnumerateArray()) + { + if (bdb.TryGetProperty("uid", out var uid) && bdb.TryGetProperty("name", out var name) + && uid.TryGetInt32(out var id) && name.GetString() is { } text) + { + results.Add((id, text)); + } + } + + return results; + } + +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs new file mode 100644 index 000000000..b9778e213 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/DatabaseShape.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The database shapes this suite provisions, described declaratively. +/// +/// +/// Shape is the axis that matters, and it is not cosmetic: measurements on 2026-08-28 showed the number of A +/// records a hostname carries follows actual proxy *placement*, giving 2 for all-nodes, 3 for +/// all-master-shards and 1 for single - and the handoff takes a different branch depending on +/// whether a live sibling address exists. So the shapes below are the reason to provision databases from inside +/// the tests at all: they turn that into a matrix axis instead of a manual sweep. +/// +public sealed record DatabaseShape( + string Label, + bool OssCluster = false, + string? ProxyPolicy = null, + bool Tls = false, + bool Replication = false, + int ShardCount = 1, + string? EndpointType = null, + string ShardsPlacement = "sparse", + string IpType = "external") +{ + /// A proxied standalone database: the shape MOVING actually fires on. + public static readonly DatabaseShape ProxiedStandalone = new("proxied-standalone", ProxyPolicy: "single"); + + /// Multiple proxies, so the hostname carries several A records and a sibling always exists. + public static readonly DatabaseShape MultiProxy = new("multi-proxy", ProxyPolicy: "all-master-shards", ShardCount: 2); + + /// OSS cluster API: the family that emits SMIGRATING/SMIGRATED instead. + public static readonly DatabaseShape OssClusterApi = new("oss-cluster", OssCluster: true, ProxyPolicy: "all-master-shards", ShardCount: 2); + + /// + /// An OSS cluster whose shards are packed, so some node holds more than one. + /// + /// + /// Exists to reach the add and slot-shuffle migrations, which need a node holding several + /// shards to have something to move. The scenario setup legs cannot produce that - they provision one shard + /// per node - so those effects looked like an environment limitation until this was tried: six shards with + /// dense placement over three nodes gives two per node, and both effects then run. + /// + /// Note it also demonstrates the placement-versus-policy point from the other direction: packed masters mean + /// all-master-shards advertises a *single* address, because the count follows placement. + /// + /// + public static readonly DatabaseShape OssClusterDense = new( + "oss-cluster-dense", OssCluster: true, ProxyPolicy: "all-master-shards", ShardCount: 6, ShardsPlacement: "dense"); + + /// + /// TLS with hostname-advertised endpoints - the documented coverage gap. + /// + /// + /// endpoint_type: ip is the interesting counterpart: with addresses advertised, a verifying client + /// cannot check identity on the targets it is told to use, because the proxy certificate carries DNS names + /// and no IP SAN. That pairing is the thing no in-process harness can honestly reproduce. + /// + public static readonly DatabaseShape TlsWithHostnames = new("tls-hostnames", OssCluster: true, Tls: true, ShardCount: 2, EndpointType: "hostname"); + + /// + /// The create_database parameters for this shape. + /// + /// + /// Goes inside a database_config wrapper when it reaches the injector - a flat payload is rejected + /// with "Invalid parameter 'database_config': got None". + /// + /// Built as data rather than as a typed record, because these are the injector's wire names and its schema + /// declares parameters as untyped. The names here are no longer guesses: they match the environment's + /// own bdb_config.json and the dbconfig the injector itself publishes as a trigger requirement + /// (GET /topology-change-standalone?effect=...), which is the authoritative list. + /// + /// + public Dictionary ToCreateParameters(string name, int port) + { + var parameters = new Dictionary + { + ["name"] = name, + ["port"] = port, + ["memory_size"] = 134_217_728, // 128MB, matching what the injector asks for in its own requirements + ["eviction_policy"] = "volatile-lru", + ["replication"] = Replication, + ["sharding"] = ShardCount > 1, + ["shards_count"] = ShardCount, + ["shards_placement"] = ShardsPlacement, + ["oss_cluster"] = OssCluster, + }; + + if (ShardCount > 1) + { + // Required whenever sharding is on: Redis Enterprise rejects the database outright with + // "Invalid sharding configuration: missing shard_key_regex". These two patterns are the standard + // pair - an explicit hash tag if present, otherwise the whole key - and are what the environment's + // own bdb_config.json uses. + parameters["shard_key_regex"] = new[] + { + new Dictionary { ["regex"] = ".*\\{(?.*)\\}.*" }, + new Dictionary { ["regex"] = "(?.*)" }, + }; + } + + // only when asked for: a database with no tls_mode serves plaintext, and setting it here without + // certificates in place produces a database nothing can connect to + if (Tls) parameters["tls_mode"] = "enabled"; + + if (OssCluster) + { + // Which addresses CLUSTER SLOTS advertises, and it is not cosmetic: the default is *internal*, so a + // cluster database created without this advertises VPC-private addresses. A client outside the VPC + // then connects to the seed, discovers nodes at 10.x, and cannot reach any of them - which is + // exactly how the first dense-cluster run failed. Distinct from EndpointType (ip versus hostname), + // which is about identity rather than routing. + parameters["oss_cluster_api_preferred_ip_type"] = IpType; + } + + if (OssCluster && EndpointType is not null) + { + // Two distinct axes, easily conflated. "endpoint_type" is ip-versus-hostname - what CLUSTER SLOTS + // advertises, and therefore whether a verifying TLS client can check identity on the targets it is + // given. "ip_type" is internal-versus-external, which is about routing rather than identity. + parameters["oss_cluster_api_preferred_endpoint_type"] = EndpointType; + } + + if (ProxyPolicy is not null) parameters["proxy_policy"] = ProxyPolicy; + + return parameters; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs new file mode 100644 index 000000000..3040ea891 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabase.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A database the environment already provisioned, read from endpoints.json. +/// +/// +/// The counterpart to creating our own. Provisioning gives control over the shape, which is what the matrix +/// needs; this gives a run against whatever the environment template made, which is what you want when the +/// question is "does any of this work at all" rather than "does it work for shape X". +/// +/// Note endpoints.json carries more than expected: each entry's raw_endpoints includes +/// proxy_policy, oss_cluster_api_preferred_endpoint_type and the address list behind the DNS +/// name. So the facts that decide client behaviour are mostly here, and the cluster REST API is only needed for +/// the explicit oss_cluster flag and the client-certificate settings. +/// +/// +public sealed record ExistingDatabase( + string Key, + int BdbId, + string Host, + int Port, + bool Tls, + string? Username, + string? Password, + string? ProxyPolicy, + string? EndpointType, + IReadOnlyList Addresses) +{ + /// + /// How many addresses the hostname is expected to resolve to. + /// + /// + /// The measured driver of handoff behaviour: with more than one address a live sibling always exists, so a + /// handoff steps sideways immediately; with one, it has to wait for the record to move. Tests that care + /// should assert on this rather than on the policy name, because the count follows actual proxy + /// *placement* - an all-master-shards database whose shards share a node advertises one address. + /// + public int AdvertisedAddressCount => Addresses.Count; + + public override string ToString() => $"{Key} ({Host}:{Port}, bdb {BdbId}, {ProxyPolicy ?? "?"}, {AdvertisedAddressCount} addr)"; + + /// + /// Reads every database in the environment's endpoints.json, keyed as that file keys them. + /// + public static Dictionary ReadAll(FaultInjectorEnvironment environment) + { + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + var path = Path.Combine(environment.ConfigDirectory.FullName, "endpoints.json"); + if (!File.Exists(path)) return results; + + using var document = JsonDocument.Parse(File.ReadAllText(path)); + foreach (var entry in document.RootElement.EnumerateObject()) + { + if (entry.Value.ValueKind != JsonValueKind.Object) continue; + if (TryRead(entry.Name, entry.Value, out var database)) results[entry.Name] = database; + } + + return results; + } + + private static bool TryRead(string key, JsonElement element, out ExistingDatabase database) + { + database = null!; + if (!element.TryGetProperty("bdb_id", out var bdbId) || !bdbId.TryGetInt32(out var id)) return false; + + // "endpoints" holds "host:port" or a redis:// URI depending on how the environment was templated + string? host = null; + int port = 0; + if (element.TryGetProperty("endpoints", out var endpoints) && endpoints.ValueKind == JsonValueKind.Array + && endpoints.GetArrayLength() > 0 && endpoints[0].GetString() is { } first) + { + var text = first; + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) text = text[(scheme + 3)..]; + var colon = text.LastIndexOf(':'); + if (colon > 0 && int.TryParse(text[(colon + 1)..], out port)) host = text[..colon]; + } + + string? proxyPolicy = null, endpointType = null; + var addresses = new List(); + if (element.TryGetProperty("raw_endpoints", out var raw) && raw.ValueKind == JsonValueKind.Array + && raw.GetArrayLength() > 0) + { + var head = raw[0]; + proxyPolicy = ReadString(head, "proxy_policy"); + endpointType = ReadString(head, "oss_cluster_api_preferred_endpoint_type"); + host ??= ReadString(head, "dns_name"); + if (port == 0 && head.TryGetProperty("port", out var rawPort)) rawPort.TryGetInt32(out port); + + if (head.TryGetProperty("addr", out var addr) && addr.ValueKind == JsonValueKind.Array) + { + foreach (var item in addr.EnumerateArray()) + { + if (item.GetString() is { } address) addresses.Add(address); + } + } + } + + if (host is null || port == 0) return false; + + database = new ExistingDatabase( + key, + id, + host, + port, + element.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, + ReadString(element, "username"), + ReadString(element, "password"), + proxyPolicy, + endpointType, + addresses); + return true; + } + + private static string? ReadString(JsonElement element, string name) + => element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null; + + /// + /// Connection options for this database, with maintenance notifications required. + /// + public ConfigurationOptions GetClientConfig(FaultInjectorEnvironment environment, MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + User = string.Equals(Username, "default", StringComparison.OrdinalIgnoreCase) ? null : Username, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = mode, + AbortOnConnectFail = false, + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Tls) + { + options.Ssl = true; + options.SslHost = Host; + var caPath = environment.CertificateAuthorityPath + ?? throw new InvalidOperationException($"{Key} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}"); + options.TrustIssuer(caPath); + } + + return options; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs new file mode 100644 index 000000000..c6ebe4453 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ExistingDatabaseFixture.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Runs against the databases the environment template already created, rather than provisioning new ones. +/// +/// +/// The cheap tier, and the right one to start from: it needs none of the create_database schema, so it +/// answers "does the client work against a real deployment" before anything depends on parameter names that are +/// documented only as prose. +/// +public class ExistingDatabaseFixture : IAsyncLifetime +{ + private FaultInjectorClient? _injector; + private string? _skipReason; + + public IReadOnlyDictionary Databases { get; private set; } + = new Dictionary(); + + public FaultInjectorEnvironment Environment => + FaultInjectorEnvironment.Current ?? throw new InvalidOperationException("no environment; call RequireAvailable first"); + + public FaultInjectorClient Injector => + _injector ?? throw new InvalidOperationException("no injector; call RequireAvailable first"); + + /// Skips the calling test when there is nothing to talk to. + public void RequireAvailable() + { + if (_skipReason is not null) Assert.Skip(_skipReason); + } + + /// + /// The named database, or a skip if this environment does not have one. + /// + /// + /// A skip rather than a failure, deliberately: which databases exist is a property of the environment + /// template somebody chose, so a template without an oss_cluster database is a reason not to run the + /// cluster tests, not evidence of a bug. + /// + public ExistingDatabase Require(string key) + { + RequireAvailable(); + if (!Databases.TryGetValue(key, out var database)) + { + Assert.Skip($"this environment has no '{key}' database (found: {string.Join(", ", Databases.Keys)})"); + } + + return database!; + } + + /// + /// Where fixture-level diagnostics go. Deliberately collected rather than written to + /// : a fixture has no test output helper, and anything it prints to the console is + /// easily lost - which is how a silently failing setup call survived a day of use. + /// + public List SetupLog { get; } = []; + + private void log(string message) + { + SetupLog.Add(message); + Console.WriteLine(message); + } + + public async ValueTask InitializeAsync() + { + if (FaultInjectorEnvironment.Current is null) + { + _skipReason = FaultInjectorEnvironment.UnavailableReason ?? "no fault-injector environment configured"; + return; + } + + if (!FaultInjectorEnvironment.IsEnabled) + { + _skipReason = "set E2E_SCENARIO_TESTS=true to run against a real deployment"; + return; + } + + _injector = new FaultInjectorClient(FaultInjectorEnvironment.Current.InjectorUrl); + Databases = ExistingDatabase.ReadAll(FaultInjectorEnvironment.Current); + await EnsureMaintenanceNotificationsEnabledAsync(); + } + + public ValueTask DisposeAsync() + { + _injector?.Dispose(); + return default; + } + + /// + /// Turns on the cluster-level maintenance-notification flags. + /// + /// + /// A different thing from the per-connection opt-in, and both are required: the cluster flag decides whether + /// CLIENT MAINT_NOTIFICATIONS exists at all, and the command opts one connection in. Without this a + /// run fails at connect - correctly, since the tests ask for + /// - but for a reason that has nothing to do with the + /// client, so it is worth setting rather than diagnosing. + /// + /// Through the injector rather than a direct REST PUT, so the change leaves an action id behind and + /// shows up in the injector's history like everything else. Best-effort: if the action type is not + /// available on this build, the connect failure that follows says so clearly enough. + /// + /// + private async Task EnsureMaintenanceNotificationsEnabledAsync() + { + // nested under "config", not flattened - the same shape create_database wants for "database_config". + // A flat payload is rejected with "Invalid parameter 'config': got None", and because this call is + // best-effort it failed *silently* for a whole day of testing without anybody noticing. The impact was + // small: the environment templates enable these flags at provision time, so this is a safety net rather + // than the mechanism, and the tests were passing on their own merits. It matters for an environment + // provisioned without them, where the alternative is every test failing at connect and blaming the + // client for a server-side setting. + var parameters = new Dictionary + { + ["config"] = new Dictionary + { + ["client_maint_notifications"] = true, // proxy-routed databases + ["oss_cluster_client_maint_notifications"] = true, // oss_cluster databases + }, + }; + + try + { + await Injector.RunActionAsync("update_cluster_config", parameters, timeout: TimeSpan.FromMinutes(2)); + log("cluster maintenance-notification flags enabled"); + } + catch (Exception ex) + { + // Still best-effort - a cluster may not support the flags at all - but no longer silent: it goes to + // the test output, where the connect failure that follows can be attributed to it. + log($"could not set cluster maintenance-notification flags: {ScenarioSupport.Summarize(ex.Message)}"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs new file mode 100644 index 000000000..6b1b18b5a --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Everything this suite needs to reach a real deployment, discovered from one directory. +/// +/// +/// One path is the whole configuration, deliberately. That directory is the one mounted into the fault +/// injector as /app/config and the one docker compose up is run from, so it already holds the +/// compose file, the cluster credentials (env_output.json), the CA certificate, and - once databases +/// exist - endpoints.json. Asking for anything else would mean the person who provisioned the +/// environment has to hand-carry facts into the test run, which is the error-prone part. +/// +/// Point at it with SER_FI_CONFIG_DIR; FI_CONSOLE_CONFIG_DIR is honoured as a fallback so this +/// runs against the same directory as the fault-injector console with no extra setup. +/// +/// +public sealed class FaultInjectorEnvironment +{ + private const string ConfigDirVariable = "SER_FI_CONFIG_DIR"; + private const string ConsoleConfigDirVariable = "FI_CONSOLE_CONFIG_DIR"; + private const string InjectorUrlVariable = "FAULT_INJECTION_API_URL"; + private const string EnabledVariable = "E2E_SCENARIO_TESTS"; + private const string DefaultInjectorUrl = "http://127.0.0.1:20324"; + + /// + /// The environment for this run, or null when none is configured. + /// + public static FaultInjectorEnvironment? Current { get; } = Discover(); + + /// + /// Why there is no environment, for a skip message that says something useful. + /// + public static string? UnavailableReason { get; private set; } + + private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) + { + ConfigDirectory = configDirectory; + InjectorUrl = injectorUrl; + } + + public DirectoryInfo ConfigDirectory { get; } + + public Uri InjectorUrl { get; } + + /// + /// The CA certificate that signs the proxy certificates, for . + /// + /// + /// The certificates are self-signed per environment, so trusting the issuer is how a TLS test validates + /// anything at all. Note what this is *not*: a switch that disables validation. A test that cannot find the + /// CA fails rather than falling back to trusting everything, because a TLS test that silently stops + /// checking identity is worse than no TLS test - it reports success for the one thing it exists to catch. + /// + public string? CertificateAuthorityPath { get; private set; } + + /// + /// Cluster credentials for the REST API on port 9443, when env_output.json carries them. + /// + /// + /// Needed for the facts endpoints.json does not carry - notably oss_cluster and the + /// advertised endpoint type - which matter because they decide which notification family a database can + /// even emit. Tests that provision their own database know what they asked for and need none of this. + /// + public ClusterCredentials? Cluster { get; private set; } + + private static FaultInjectorEnvironment? Discover() + { + var path = Environment.GetEnvironmentVariable(ConfigDirVariable) + ?? Environment.GetEnvironmentVariable(ConsoleConfigDirVariable); + + if (string.IsNullOrWhiteSpace(path)) + { + UnavailableReason = $"no fault-injector environment: set {ConfigDirVariable} to the directory holding env_output.json"; + return null; + } + + var directory = new DirectoryInfo(path); + if (!directory.Exists) + { + // configured but wrong is a mistake worth reporting, not a reason to quietly do nothing + UnavailableReason = $"{ConfigDirVariable} points at '{path}', which does not exist"; + return null; + } + + var url = Environment.GetEnvironmentVariable(InjectorUrlVariable); + var injectorUrl = Uri.TryCreate(url, UriKind.Absolute, out var parsed) ? parsed : new Uri(DefaultInjectorUrl); + + var result = new FaultInjectorEnvironment(directory, injectorUrl) + { + CertificateAuthorityPath = FindCertificateAuthority(directory), + }; + result.Cluster = ReadClusterCredentials(directory); + return result; + } + + /// + /// Whether the caller actually meant to run destructive tests against a real deployment. + /// + /// + /// Two gates rather than one, and they mean different things. Without a config directory there is nothing + /// to talk to, which is the ordinary state for everybody else and skips. With a directory but without + /// E2E_SCENARIO_TESTS=true we also skip, because these tests create and delete databases and nobody + /// should trip that by running the full traversal. What must *not* happen is a third state where the + /// environment is configured, meant, and broken, yet the run still reports success - see + /// . + /// + public static bool IsEnabled => + string.Equals(Environment.GetEnvironmentVariable(EnabledVariable), "true", StringComparison.OrdinalIgnoreCase); + + private static string? FindCertificateAuthority(DirectoryInfo directory) + { + // names seen across the environment templates, most specific first + foreach (var candidate in new[] { "ca.pem", "ca.crt", "proxy_cert.pem", "redislabs_ca.pem" }) + { + var path = Path.Combine(directory.FullName, candidate); + if (File.Exists(path)) return path; + } + + return null; + } + + private static ClusterCredentials? ReadClusterCredentials(DirectoryInfo directory) + { + var path = Path.Combine(directory.FullName, "env_output.json"); + if (!File.Exists(path)) return null; + + try + { + using var document = JsonDocument.Parse(File.ReadAllText(path)); + + // two schemas in the wild: single-cluster templates put outputs at the top level, and the AWS + // multi-cluster template nests them under .clusters.value[N]. Both are handled because which one + // you get is a property of the template somebody chose, not of anything a test can control. + var root = document.RootElement; + if (root.TryGetProperty("clusters", out var clusters) + && clusters.TryGetProperty("value", out var values) + && values.ValueKind == JsonValueKind.Array + && values.GetArrayLength() > 0) + { + root = values[0]; + } + + var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); + var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); + var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + + return name is null || user is null || password is null ? null : new ClusterCredentials(name, user, password); + } + catch (Exception) + { + // a malformed env_output.json is not fatal here: only the tests that need REST enrichment care, + // and they report it themselves rather than failing every test in the suite at discovery time + return null; + } + } + + /// + /// Reads a property that may be a bare value or a terraform-style { "value": ... } wrapper. + /// + private static string? ReadValue(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var value)) return null; + if (value.ValueKind == JsonValueKind.Object && value.TryGetProperty("value", out var inner)) value = inner; + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + public sealed record ClusterCredentials(string ClusterName, string Username, string Password) + { + public Uri RestUrl => new($"https://{ClusterName}:9443"); + + /// Never let the password reach a log; the endpoints are real and reachable. + public override string ToString() => $"{Username}@{ClusterName}"; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs new file mode 100644 index 000000000..8ea143f8e --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -0,0 +1,217 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Provisions one database shape for the tests that need it, and takes it away afterwards. +/// +/// +/// Per *shape*, not per test: creating a database on a real cluster is slow, so several test classes share one. +/// +/// The three-state behaviour matters more than the provisioning. No environment configured, or the +/// E2E_SCENARIO_TESTS gate not set, means every test skips - which is the ordinary case for anybody +/// running the full traversal, and these tests create and delete real databases. But an environment that *is* +/// configured and meant, and then does not work, must **fail**: a suite that skips on a broken environment +/// reports success for tests that never ran, and you will trust it exactly when you should not. So the absent +/// case skips from the test body, and the broken case throws from here. +/// +/// +public abstract class FaultInjectorFixture(DatabaseShape shape) : IAsyncLifetime +{ + /// + /// Shared by every database this suite creates, in every run. + /// + /// + /// The sweep that cleans up leaks from earlier runs matches on this and nothing else, so it can never touch + /// a database somebody created by hand. Worth being strict about: the alternative is a test suite that + /// deletes production-shaped things on a shared cluster. + /// + public const string NamePrefix = "sertest-"; + + /// + /// Name prefixes the sweep will remove, beyond our own. + /// + /// + /// A scenario's setup leg creates its *own* database, named by the injector - tcs- for + /// topology-change, sm- for slot-migrate - so those leaks are not ours to name but are ours to clean + /// up. And they do leak: cancelling the setup request does not cancel the injector's work, so a run killed + /// mid-setup leaves a database nothing holds a handle to. That happened, 22 times, and needed clearing by + /// hand because the sweep only knew about databases we had created ourselves. + /// + /// Safe on the assumption this is a dedicated test environment - which the tier already assumes, since it + /// creates and destroys databases and reshapes the cluster. Databases named in endpoints.json (the + /// template's own) are never touched, whatever their prefix. + /// + /// + private static readonly string[] SweepablePrefixes = [NamePrefix, "tcs-", "sm-"]; + + private static readonly string RunId = Guid.NewGuid().ToString("n")[..6]; + + private FaultInjectorClient? _injector; + private string? _skipReason; + + public DatabaseShape Shape { get; } = shape; + + /// The database created for this fixture, once has run. + public ProvisionedDatabase? Database { get; private set; } + + public FaultInjectorEnvironment Environment => + FaultInjectorEnvironment.Current ?? throw new InvalidOperationException("no environment; call RequireAvailable first"); + + public FaultInjectorClient Injector => + _injector ?? throw new InvalidOperationException("no injector; call RequireAvailable first"); + + /// + /// Skips the calling test when there is no environment to talk to. + /// + public void RequireAvailable() + { + if (_skipReason is not null) Assert.Skip(_skipReason); + } + + public async ValueTask InitializeAsync() + { + if (FaultInjectorEnvironment.Current is null) + { + _skipReason = FaultInjectorEnvironment.UnavailableReason ?? "no fault-injector environment configured"; + return; + } + + if (!FaultInjectorEnvironment.IsEnabled) + { + _skipReason = "set E2E_SCENARIO_TESTS=true to run against a real deployment (these tests create and delete databases)"; + return; + } + + // configured and meant: from here on, problems are failures + _injector = new FaultInjectorClient(FaultInjectorEnvironment.Current.InjectorUrl); + await SweepOrphansAsync(); + Database = await CreateDatabaseAsync(); + } + + public async ValueTask DisposeAsync() + { + // unconditionally, and before anything else can throw: a database left behind holds a port and a slice + // of cluster memory, and the next run collides with it + if (Database is { } database && _injector is { } injector) + { + try + { + await injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = database.BdbId }); + } + catch (Exception ex) + { + // never mask a test failure with a cleanup failure; the sweep will get it next time + Console.WriteLine($"failed to delete {database.Name} (bdb {database.BdbId}): {ex.Message}"); + } + } + + _injector?.Dispose(); + } + + /// + /// Creates the database for this shape, working around port collisions the way go-redis has to. + /// + /// + /// Port collisions are common enough on a shared cluster that go-redis carries a dedicated + /// CreateDatabaseWithPortRetry helper; this is the same idea. The base port is deliberately high and + /// the walk is upward, so a collision costs one attempt rather than a failed run. + /// + private async Task CreateDatabaseAsync() + { + // Well clear of the 13xxx range the scenario setups pick from, so our own databases are not competing + // with theirs for ports in the first place. + const int BasePort = 14500, Attempts = 8; + var name = $"{NamePrefix}{Shape.Label}-{RunId}"; + Exception? last = null; + + for (int attempt = 0; attempt < Attempts; attempt++) + { + var port = BasePort + (attempt * 3); + try + { + // nested under database_config, not flattened: the injector answers a flat payload with + // "Invalid parameter 'database_config': got None" + var result = await Injector.RunActionAsync( + "create_database", + new Dictionary { ["database_config"] = Shape.ToCreateParameters(name, port) }); + return ProvisionedDatabase.FromCreateResult(name, port, Shape, result, Environment); + } + catch (Exception ex) when (IsPortCollision(ex)) + { + last = ex; + Console.WriteLine($"create_database on port {port} collided, retrying higher"); + } + } + + throw new InvalidOperationException($"could not create '{name}' after {Attempts} attempts", last); + } + + /// + /// Whether a create failure is worth trying a different port for. + /// + /// + /// Only port collisions are: everything else - a malformed config, a cluster with no capacity - fails + /// identically on every port, and retrying eight times turns one clear error message into eight and delays + /// the report by half a minute. The first version of this retried everything, and buried + /// "missing shard_key_regex" under eight identical tracebacks. + /// + private static bool IsPortCollision(Exception ex) => + // "port_unavailable" is what Redis Enterprise actually answers, with the prose "Unavailable or invalid + // port" - which matched none of the phrases the first version of this looked for, so a perfectly + // retryable collision failed the whole fixture instead of moving up a port. + ex.Message.Contains("port_unavailable", StringComparison.OrdinalIgnoreCase) + || (ex.Message.Contains("port", StringComparison.OrdinalIgnoreCase) + && (ex.Message.Contains("in use", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("already", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("taken", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("unavailable", StringComparison.OrdinalIgnoreCase) + || ex.Message.Contains("conflict", StringComparison.OrdinalIgnoreCase))); + + /// + /// Best-effort removal of databases left behind by earlier runs. + /// + /// + /// Best-effort on purpose: listing databases needs the cluster REST API, which needs credentials this + /// directory may not carry, and being unable to tidy up is not a reason to fail a run that could otherwise + /// proceed. The per-fixture delete in is the reliable path; this only catches + /// leaks from a run that was killed. + /// + private async Task SweepOrphansAsync() + { + var cluster = Environment.Cluster; + if (cluster is null) + { + Console.WriteLine("no cluster credentials in env_output.json; skipping orphan sweep"); + return; + } + + try + { + // Never touch the environment's own databases. Matched by bdb id rather than by name: endpoints.json + // keys them by the name they were configured with, and relying on that key equalling the database's + // actual name is an assumption worth not making when the consequence is deleting the wrong thing. + var template = ExistingDatabase.ReadAll(Environment).Values.Select(d => d.BdbId).ToHashSet(); + + using var rest = new ClusterRestClient(cluster, Environment.CertificateAuthorityPath); + foreach (var (bdbId, name) in await rest.ListDatabasesAsync()) + { + if (template.Contains(bdbId)) continue; + if (!SweepablePrefixes.Any(prefix => name.StartsWith(prefix, StringComparison.Ordinal))) continue; + + Console.WriteLine($"sweeping orphaned test database {name} (bdb {bdbId})"); + await Injector.RunActionAsync("delete_database", new Dictionary { ["bdb_id"] = bdbId }); + } + } + catch (Exception ex) + { + Console.WriteLine($"orphan sweep skipped: {ex.Message}"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs new file mode 100644 index 000000000..a6268eb87 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ProvisionedDatabase.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A database this suite created, and how to connect to it. +/// +/// +/// Note what this type does *not* need to do: read endpoints.json to discover whether the database is +/// oss_cluster, or what endpoint type it advertises. A test that provisioned the database knows what it +/// asked for, which is the real argument for provisioning from inside the suite - the alternative is carrying +/// those facts from whoever set the environment up into the run, by hand. +/// +public sealed class ProvisionedDatabase +{ + private ProvisionedDatabase(string name, int bdbId, string host, int port, DatabaseShape shape, FaultInjectorEnvironment environment) + { + Name = name; + BdbId = bdbId; + Host = host; + Port = port; + Shape = shape; + Environment = environment; + } + + public string Name { get; } + + public int BdbId { get; } + + public string Host { get; } + + public int Port { get; } + + public DatabaseShape Shape { get; } + + public FaultInjectorEnvironment Environment { get; } + + public string? Password { get; private init; } + + /// + /// Connection options for this database, with maintenance notifications requested. + /// + /// + /// rather than Auto: in a test, a server that + /// silently declines the opt-in should fail the connection loudly rather than produce a run that passes + /// while observing nothing. That is the opposite of the right default for production, and exactly right + /// here. + /// + public ConfigurationOptions GetClientConfig() + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = MaintenanceNotificationMode.Enabled, + AbortOnConnectFail = false, + // real network, real cluster: connect and command budgets have to tolerate a WAN round trip, and a + // cluster that is mid-scenario is slower still + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Shape.Tls) + { + options.Ssl = true; + options.SslHost = Host; + + // The certificates are self-signed per environment, so trusting the issuer is what makes a TLS test + // mean anything. If the CA is missing we fail here rather than disabling validation: a TLS test that + // quietly stops checking identity reports success for the one thing it exists to catch. + var caPath = Environment.CertificateAuthorityPath + ?? throw new InvalidOperationException( + $"{Shape.Label} needs TLS, but no CA certificate was found in {Environment.ConfigDirectory.FullName}; " + + "validation is not disabled for tests"); + options.TrustIssuer(caPath); + } + + return options; + } + + /// Masked, because these endpoints are real and reachable from the internet. + public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, {Shape.Label})"; + + /// + /// Reads what the injector reported back after create_database. + /// + /// + /// Tolerant by design: the response shape is documented as prose, and the fields worth having may arrive at + /// the top level or nested under an output/result object. What cannot be guessed is the bdb_id - + /// without it there is nothing to delete afterwards - so that one is required and its absence is loud. + /// + public static ProvisionedDatabase FromCreateResult( + string name, + int requestedPort, + DatabaseShape shape, + JsonElement result, + FaultInjectorEnvironment environment) + { + var bdbId = FindInt(result, "bdb_id") + ?? throw new InvalidOperationException($"create_database for '{name}' returned no bdb_id: {result}"); + + var host = FindString(result, "endpoint") ?? FindString(result, "dns_name") ?? environment.Cluster?.ClusterName + ?? throw new InvalidOperationException($"create_database for '{name}' returned no endpoint, and env_output.json names no cluster: {result}"); + + // an endpoint may arrive as "host:port" or as a bare host + var port = requestedPort; + var colon = host.LastIndexOf(':'); + if (colon > 0 && int.TryParse(host[(colon + 1)..], out var parsedPort)) + { + port = parsedPort; + host = host[..colon]; + } + + return new ProvisionedDatabase(name, bdbId, host, port, shape, environment) + { + Password = FindString(result, "password"), + }; + } + + private static int? FindInt(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.Number && candidate.TryGetInt32(out var value)) return value; + if (candidate.ValueKind == JsonValueKind.String && int.TryParse(candidate.GetString(), out var parsed)) return parsed; + } + + return null; + } + + private static string? FindString(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.String) return candidate.GetString(); + if (candidate.ValueKind == JsonValueKind.Array && candidate.GetArrayLength() > 0 && candidate[0].ValueKind == JsonValueKind.String) + { + return candidate[0].GetString(); // endpoints arrive as a list often enough + } + } + + return null; + } + + /// + /// Yields every value for a property name, at any depth - shallowest first. + /// + private static IEnumerable Walk(JsonElement element, string name) + { + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty(name, out var direct)) yield return direct; + + foreach (var property in element.EnumerateObject()) + { + foreach (var nested in Walk(property.Value, name)) yield return nested; + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + foreach (var nested in Walk(item, name)) yield return nested; + } + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs new file mode 100644 index 000000000..d467ba96c --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/FaultInjectorClient.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The fault injector's HTTP surface, as much of it as this suite needs. +/// +/// +/// Deliberately the same shape go-redis, redis-py and node-redis wrap: POST /action returning an id and +/// GET /action/{id} to poll. They integrated independently and converged, so the contract is stable +/// enough to depend on, and their scenario documentation transfers to this client. +/// +public sealed class FaultInjectorClient(Uri baseAddress) : IDisposable +{ + /// + /// No , deliberately. The injector *queues* actions, so a request can + /// legitimately sit for many minutes when anything else is in flight - and a two-minute limit here failed + /// twenty of twenty-six tests the first time the whole suite ran together, all of them reported as a + /// cancelled HTTP request rather than as what they were. Bounding belongs to the caller's cancellation + /// token and to , which can tell "still working" from "wedged"; a blanket + /// client timeout cannot. + /// + private readonly HttpClient _http = new() { BaseAddress = baseAddress, Timeout = Timeout.InfiniteTimeSpan }; + + /// + /// Statuses that mean "still going". Both of them, which is the point. + /// + /// + /// A job passes through pending *and* running, so a loop that waits only on pending + /// returns while the work is still in flight - and then the test asserts against a cluster that has not + /// finished changing. Documented as a trap in the console's notes; encoded here so it cannot be + /// rediscovered. + /// + private static readonly HashSet PendingStatuses = new(StringComparer.OrdinalIgnoreCase) { "pending", "running", "in_progress" }; + + private static readonly HashSet FailedStatuses = new(StringComparer.OrdinalIgnoreCase) { "failed", "cancelled", "error" }; + + public void Dispose() => _http.Dispose(); + + /// + /// Fires an action and returns its id, without waiting. + /// + public async Task StartActionAsync(string type, object? parameters = null, CancellationToken cancellationToken = default) + { + var payload = new ActionRequest(type, parameters); + using var response = await _http.PostAsJsonAsync("/action", payload, cancellationToken); + await EnsureSuccessAsync(response, $"POST /action ({type})", cancellationToken); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + return ReadActionId(document.RootElement) + ?? throw new InvalidOperationException($"the injector accepted '{type}' but returned no action id: {document.RootElement}"); + } + + /// + /// Fires an action and waits for it to finish, returning its final payload. + /// + public async Task RunActionAsync( + string type, + object? parameters = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var id = await StartActionAsync(type, parameters, cancellationToken); + return await WaitForActionAsync(id, timeout, cancellationToken); + } + + /// + /// Polls an action to completion. + /// + /// + /// The timeout is generous by default because these actions move data: a slot migration or a node coming + /// out of maintenance mode is minutes, not seconds. A tight default here would produce failures that look + /// like product bugs. + /// + public async Task WaitForActionAsync( + string id, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromMinutes(10)); + string? lastStatus = null; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + using var response = await _http.GetAsync($"/action/{id}", cancellationToken); + await EnsureSuccessAsync(response, $"GET /action/{id}", cancellationToken); + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(body); + var status = ReadStatus(document.RootElement); + lastStatus = status ?? lastStatus; + + if (status is not null && !PendingStatuses.Contains(status)) + { + if (FailedStatuses.Contains(status)) + { + throw new InvalidOperationException($"fault-injector action {id} ended as '{status}': {body}"); + } + + // clone: the JsonDocument is disposed with this scope, and callers keep the result + return document.RootElement.Clone(); + } + + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"fault-injector action {id} was still '{lastStatus ?? "unknown"}' after {(timeout ?? TimeSpan.FromMinutes(10)).TotalSeconds:0}s"); + } + + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + } + + /// + /// Discovers which triggers are valid for an effect. + /// + /// + /// Worth asking rather than hardcoding: the effect/trigger matrix is sparse - maintenance_mode only + /// supports remove-add and remove, failover needs replication enabled - and some + /// scenarios do not enumerate their triggers in the schema at all. + /// + public async Task GetValidTriggersAsync(string scenario, string effect, int clusterIndex = 0, CancellationToken cancellationToken = default) + { + using var response = await _http.GetAsync($"/{scenario}?effect={Uri.EscapeDataString(effect)}&cluster_index={clusterIndex}", cancellationToken); + await EnsureSuccessAsync(response, $"GET /{scenario}?effect={effect}", cancellationToken); + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + return document.RootElement.Clone(); + } + + /// + /// Runs a scenario's setup / run / teardown triple's individual legs. + /// + public async Task PostScenarioAsync( + string scenario, + string? leg, + IReadOnlyDictionary query, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var path = leg is null ? $"/{scenario}" : $"/{scenario}/{leg}"; + var separator = '?'; + foreach (var pair in query) + { + if (pair.Value is null) continue; + path += $"{separator}{pair.Key}={Uri.EscapeDataString(pair.Value)}"; + separator = '&'; + } + + using var response = await _http.PostAsync(path, content: null, cancellationToken); + await EnsureSuccessAsync(response, $"POST {path}", cancellationToken); + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(body); + + // the scenario legs only *enqueue*; a caller that returns here is racing the work it asked for + var id = ReadActionId(document.RootElement); + return id is null ? document.RootElement.Clone() : await WaitForActionAsync(id, timeout, cancellationToken); + } + + private static async Task EnsureSuccessAsync(HttpResponseMessage response, string what, CancellationToken cancellationToken) + { + if (response.IsSuccessStatusCode) return; + + // include the body: the injector explains refusals there, and "400 Bad Request" alone has cost people + // hours on the effect/trigger matrix + var body = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"{what} failed: {(int)response.StatusCode} {response.ReasonPhrase}. {body}"); + } + + /// + /// The id of a pollable action, if this response describes one. + /// + /// + /// Deliberately not setup_id: a scenario's setup returns a *handle* to state held in the injector, + /// which is passed back on the run and teardown legs, and is not something /action/{id} knows about. + /// Treating it as an action id polls a URL that does not exist. + /// + private static string? ReadActionId(JsonElement element) + { + foreach (var name in new[] { "action_id", "id" }) + { + if (element.TryGetProperty(name, out var value)) + { + return value.ValueKind switch + { + JsonValueKind.String => value.GetString(), + JsonValueKind.Number => value.ToString(), + _ => null, + }; + } + } + + return null; + } + + private static string? ReadStatus(JsonElement element) + => element.TryGetProperty("status", out var status) && status.ValueKind == JsonValueKind.String + ? status.GetString() + : null; + + private sealed record ActionRequest( + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("parameters")] object? Parameters); +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs new file mode 100644 index 000000000..c3b7e1b96 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioRun.cs @@ -0,0 +1,290 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// One run of a fault-injector scenario: setup, fire, tear down. +/// +/// +/// The setup leg provisions its own database, which is the important part: every trigger publishes the +/// dbconfig it needs (GET /topology-change-standalone?effect=...), and all of them want +/// proxy_policy: single - a shape the environment templates do not create. So a scenario cannot be +/// pointed at an existing database, and there is no need to hand-roll create_database for one either. +/// +/// Teardown is the whole reason this is a type rather than three calls. A scenario left set up holds cluster +/// state - the flags it enabled, the database it made, nodes it excluded - and poisons every run after it. +/// +/// +public sealed class ScenarioRun : IAsyncDisposable +{ + private readonly FaultInjectorClient _injector; + private readonly string _scenario; + private readonly Action _log; + + private ScenarioRun(FaultInjectorClient injector, string scenario, string effect, string trigger, Action log) + { + _injector = injector; + _scenario = scenario; + Effect = effect; + Trigger = trigger; + _log = log; + } + + public string Effect { get; } + + public string Trigger { get; } + + /// The setup handle, passed back on the run and teardown legs. + public string? SetupId { get; private set; } + + /// The database the setup leg created, when it says which. + public int? BdbId { get; private set; } + + /// Whatever setup returned, for a test that wants to look at more than we model. + public JsonElement SetupResult { get; private set; } + + /// The database setup created, ready to connect to. + /// + /// Setup hands back everything needed - endpoint, password, TLS - so a scenario test never has to look in + /// endpoints.json or ask the cluster REST API. Verified against a live run: the response carries + /// setup_id, bdb_id, db_name, endpoints, password, tls, + /// mtls_files and config (the proxy policy it chose to satisfy the trigger). + /// + public ScenarioDatabase? Database { get; private set; } + + /// The client-side material an mTLS database requires. + /// + /// Note these are the *client* identity, issued by the fault injector's own intermediate CA, and are a + /// different trust root from the certificate that validates the server: that one comes from the + /// environment's ca.crt. Conflating the two is the obvious way to get an mTLS test wrong. + /// The paths the setup response gives are relative to the config directory. + /// + public sealed record MtlsMaterial(string ClientCertificatePath, string ClientKeyPath, string CaChainPath); + + /// The database a scenario provisioned for itself. + public sealed record ScenarioDatabase(string Name, int BdbId, string Host, int Port, bool Tls, string? Password, string? ProxyPolicy, MtlsMaterial? Mtls = null) + { + public override string ToString() => $"{Name} ({Host}:{Port}, bdb {BdbId}, policy {ProxyPolicy ?? "?"})"; + + public ConfigurationOptions GetClientConfig( + FaultInjectorEnvironment environment, + MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var options = new ConfigurationOptions + { + EndPoints = { { Host, Port } }, + Password = Password, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = mode, + AbortOnConnectFail = false, + ConnectTimeout = 15_000, + SyncTimeout = 15_000, + }; + + if (Tls) + { + options.Ssl = true; + options.SslHost = Host; + options.TrustIssuer(environment.CertificateAuthorityPath + ?? throw new InvalidOperationException($"{Name} uses TLS but no CA certificate was found in {environment.ConfigDirectory.FullName}")); + + // ...and if the database demands a client certificate, present one. Deliberately after + // TrustIssuer: that decides whether we accept the *server*, this decides what we offer about + // ourselves, and they use different trust roots. + if (Mtls is { } mtls) options.SetUserPemCertificate(mtls.ClientCertificatePath, mtls.ClientKeyPath); + } + + return options; + } + } + + /// + /// Sets a scenario up, ready to fire. + /// + /// + /// The trigger for the *setup* leg, when it differs from the one being fired. + /// + /// + /// The two triggers are different questions, which the naming hides. The setup leg's trigger says how to + /// *provision* - /slot-migrate/setup accepts only reshard, because provisioning a cluster + /// database is all it does - while the run leg's trigger says how to cause the effect (migrate, + /// maintenance_mode, failover). Passing the run trigger to setup earns a + /// "Trigger 'migrate' is not supported by slot-migrate/setup (only 'reshard')". + /// + public static async Task SetupAsync( + FaultInjectorClient injector, + string scenario, + string effect, + string trigger, + Action log, + IReadOnlyDictionary? extra = null, + string? setupTrigger = null, + CancellationToken cancellationToken = default) + { + var run = new ScenarioRun(injector, scenario, effect, trigger, log); + var query = new Dictionary { ["effect"] = effect, ["trigger"] = setupTrigger ?? trigger }; + if (extra is not null) + { + foreach (var pair in extra) query[pair.Key] = pair.Value; + } + + log($"setup {scenario}: effect={effect} setup-trigger={setupTrigger ?? trigger} (firing '{trigger}')"); + run.SetupResult = await injector.PostScenarioAsync(scenario, "setup", query, cancellationToken: cancellationToken); + run.SetupId = FindString(run.SetupResult, "setup_id"); + run.BdbId = FindInt(run.SetupResult, "bdb_id"); + run.Database = ReadDatabase(run.SetupResult, run.BdbId, FaultInjectorEnvironment.Current?.ConfigDirectory.FullName); + log($"setup complete: setup_id={run.SetupId ?? "(none)"} database={run.Database?.ToString() ?? "(none)"}"); + return run; + } + + /// + /// Fires the scenario and waits for the injector to finish its work. + /// + /// + /// Note "finished" here means the injector has done what it was asked, not that the deployment has settled: + /// the notifications, the DNS change and the socket close all trail it by seconds. Callers should watch for + /// what they expect rather than assuming completion means arrival. + /// + public async Task FireAsync(CancellationToken cancellationToken = default) + { + var query = new Dictionary + { + ["effect"] = Effect, + ["trigger"] = Trigger, + ["setup_id"] = SetupId, + ["bdb_id"] = BdbId?.ToString(), + }; + + _log($"firing {_scenario}"); + var result = await _injector.PostScenarioAsync(_scenario, leg: null, query, cancellationToken: cancellationToken); + _log($"fired: {result}"); + return result; + } + + public async ValueTask DisposeAsync() + { + var query = new Dictionary + { + // both, because setup_id lives in the injector's memory and is lost if it restarts, at which point + // bdb_id is the only handle left + ["setup_id"] = SetupId, + ["bdb_id"] = BdbId?.ToString(), + ["restore_nodes"] = "true", // put back anything the scenario excluded, or the next run starts degraded + }; + + try + { + // Its own budget rather than the caller's token: teardown has to happen even when the test was + // cancelled or timed out, which is exactly when the caller's token is already dead. + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await _injector.PostScenarioAsync(_scenario, "teardown", query, cancellationToken: timeout.Token); + _log("teardown complete"); + } + catch (Exception ex) + { + // loud, but not an exception: a teardown failure must not replace the test's own verdict + _log($"TEARDOWN FAILED ({ex.Message}) - the cluster may be left with a scenario set up, and " + + $"setup_id={SetupId ?? "(none)"} bdb_id={BdbId?.ToString() ?? "(none)"} is what to clean up by hand"); + } + } + + /// + /// The mTLS material a setup reported, resolved against the config directory. + /// + private static MtlsMaterial? ReadMtls(JsonElement setup, string? configDirectory) + { + if (!setup.TryGetProperty("mtls_files", out var files) || files.ValueKind != JsonValueKind.Object) return null; + + var cert = FindString(files, "client_cert"); + var key = FindString(files, "client_key"); + var chain = FindString(files, "ca_chain"); + if (cert is null || key is null || chain is null || configDirectory is null) return null; + + return new MtlsMaterial( + System.IO.Path.Combine(configDirectory, cert), + System.IO.Path.Combine(configDirectory, key), + System.IO.Path.Combine(configDirectory, chain)); + } + + private static ScenarioDatabase? ReadDatabase(JsonElement setup, int? bdbId, string? configDirectory) + { + if (bdbId is not { } id) return null; + + var endpoint = FindString(setup, "endpoints"); + if (endpoint is null) return null; + + var text = endpoint; + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) text = text[(scheme + 3)..]; + var colon = text.LastIndexOf(':'); + if (colon <= 0 || !int.TryParse(text[(colon + 1)..], out var port)) return null; + + return new ScenarioDatabase( + FindString(setup, "db_name") ?? $"bdb-{id}", + id, + text[..colon], + port, + setup.TryGetProperty("tls", out var tls) && tls.ValueKind == JsonValueKind.True, + FindString(setup, "password"), + FindString(setup, "config"), + ReadMtls(setup, configDirectory)); + } + + private static string? FindString(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.String) return candidate.GetString(); + if (candidate.ValueKind == JsonValueKind.Number) return candidate.ToString(); + if (candidate.ValueKind == JsonValueKind.Array && candidate.GetArrayLength() > 0 + && candidate[0].ValueKind == JsonValueKind.String) + { + return candidate[0].GetString(); // "endpoints" is a list even when there is one + } + } + + return null; + } + + private static int? FindInt(JsonElement element, string name) + { + foreach (var candidate in Walk(element, name)) + { + if (candidate.ValueKind == JsonValueKind.Number && candidate.TryGetInt32(out var value)) return value; + if (candidate.ValueKind == JsonValueKind.String && int.TryParse(candidate.GetString(), out var parsed)) return parsed; + } + + return null; + } + + /// + /// Yields every value for a property name at any depth, shallowest first. + /// + /// + /// Tolerant on purpose: the scenario responses are documented as prose and nest differently between legs, so + /// searching beats asserting a shape - and a test that cannot find setup_id still has a logged + /// payload to work from rather than a deserialization error. + /// + private static IEnumerable Walk(JsonElement element, string name) + { + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty(name, out var direct)) yield return direct; + foreach (var property in element.EnumerateObject()) + { + foreach (var nested in Walk(property.Value, name)) yield return nested; + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + foreach (var nested in Walk(item, name)) yield return nested; + } + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs new file mode 100644 index 000000000..9886ee68b --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/FaultInjector/ScenarioSupport.cs @@ -0,0 +1,95 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Telling "this deployment cannot do that" apart from "the client got it wrong". +/// +/// +/// The effect/trigger matrix is sparse in ways that depend on the *cluster*, not just the database: an +/// add migration needs a node holding more than one shard, and a three-node cluster with sparse shard +/// placement gives every node exactly one. The injector reports that as a failed action with a Python +/// traceback, which is indistinguishable from a real fault unless you read the message. +/// +/// So these skip, narrowly and by message. A broad catch here would hide genuine injector failures, which is +/// the thing this tier exists to surface - hence matching on the specific condition rather than on the +/// exception type. +/// +/// +internal static class ScenarioSupport +{ + private static readonly string[] PlacementLimitations = + [ + "No node with multiple shards found", + "not enough nodes", + "no empty node", + ]; + + /// + /// Skips when the setup could not produce a database the effect can act on. + /// + public static void RequireEffectIsAchievable(ScenarioRun scenario, string effect) + { + if (scenario.Database is null) + { + Assert.Skip($"the injector did not provision a database for '{effect}'"); + } + } + + /// + /// Fires a scenario, skipping rather than failing when the cluster's shape cannot produce the effect. + /// + public static async Task FireOrSkipAsync(ScenarioRun scenario, string effect, CancellationToken cancellationToken) + { + try + { + await scenario.FireAsync(cancellationToken); + } + catch (Exception ex) when (IsPlacementLimitation(ex)) + { + Assert.Skip($"this cluster cannot produce '{effect}': {Summarize(ex.Message)}"); + } + } + + private static bool IsPlacementLimitation(Exception ex) + { + foreach (var limitation in PlacementLimitations) + { + if (ex.Message.Contains(limitation, StringComparison.OrdinalIgnoreCase)) return true; + } + + return false; + } + + /// + /// The part of a Python traceback that says what actually went wrong. + /// + /// + /// Note the traceback arrives inside a JSON string, so its line breaks are the two characters + /// \n rather than real newlines - splitting on '\n' alone matches nothing and returns + /// the whole wall of text, which is how the first version of this behaved. The injector helpfully ends with + /// "Caused by: ...", which is the only part worth putting in a skip message. + /// + internal static string Summarize(string message) + { + var caused = message.LastIndexOf("Caused by:", StringComparison.Ordinal); + if (caused >= 0) + { + var tail = message[caused..]; + var end = tail.IndexOf("\\n", StringComparison.Ordinal); + return end > 0 ? tail[..end] : tail; + } + + var lines = message.Replace("\\n", "\n", StringComparison.Ordinal) + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + for (int i = lines.Length - 1; i >= 0; i--) + { + if (lines[i].Contains("Exception:", StringComparison.Ordinal)) return lines[i]; + } + + return lines.Length > 0 ? lines[^1] : message; + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs new file mode 100644 index 000000000..9221bc7ad --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingEndpointTypeScenarioTests.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Does asking for a moving-endpoint-type make the server name a replacement? +/// +/// +/// The experiment behind an open question. Eleven MOVING notifications observed on Redis Enterprise +/// 8.0.22 all carried an explicit null for the replacement address, including cases where the server had +/// already chosen the replacement node - and every one of those was requested with a bare +/// CLIENT MAINT_NOTIFICATIONS ON. So either this build never populates the field, or the server default +/// amounts to none and we were being given exactly what we asked for. +/// +/// This distinguishes those two, which matters for more than tidiness: a named successor would let a handoff +/// skip the DNS wait entirely, and DNS has been measured trailing the notification by up to 18.7s against a +/// socket that closed at 15.7s. It also decides whether the named-successor code path is reachable at all, or +/// exists only because the contract mentions it. +/// +/// +/// Deliberately reports rather than asserts a populated address: "this build does not populate it" is a +/// legitimate answer, and the test's job is to record which answer we got, per endpoint type. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "moving-endpoint-type")] +public class MovingEndpointTypeScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + // Auto over a real socket to a public address with no TLS should derive external-ip, so the server should + // name an address rather than a hostname - which is the end-to-end proof of the derivation. + [InlineData(MaintenanceEndpointType.Auto)] + [InlineData(MaintenanceEndpointType.ServerDefault)] + [InlineData(MaintenanceEndpointType.ExternalFqdn)] + [InlineData(MaintenanceEndpointType.ExternalIp)] + [InlineData(MaintenanceEndpointType.InternalFqdn)] + [InlineData(MaintenanceEndpointType.InternalIp)] + public async Task DoesTheServerNameAReplacement(MaintenanceEndpointType type) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", "conn_drop", "endpoint_rebind", log.WriteLine, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var config = database.GetClientConfig(fixture.Environment, MaintenanceNotificationMode.Auto); + config.MaintenanceMovingEndpointType = type; + + var clock = Stopwatch.StartNew(); + var moving = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + + // Auto rather than Enabled, because a server that rejects an endpoint type we asked for is one of the + // outcomes being measured - and it should not fail the run. + log.WriteLine($"requested {type}; opt-in active = {endpoint.MaintenanceNotificationsActive}"); + if (!endpoint.MaintenanceNotificationsActive) + { + log.WriteLine("=> the server refused this endpoint type; nothing further to observe"); + return; + } + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} {push.RawMessage}"); + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + lock (moving) moving.Add(push); + } + } + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + lock (moving) + { + if (moving.Count > 0) break; + } + + await Task.Delay(500, cancellationToken); + } + + lock (moving) + { + if (moving.Count == 0) + { + log.WriteLine("=> no MOVING arrived at all"); + return; + } + + foreach (var push in moving) + { + log.WriteLine($"=> {type}: NewEndPoint = {push.NewEndPoint?.ToString() ?? "(null)"}; payload = {push.Payload ?? "(null)"}"); + } + + if (type == MaintenanceEndpointType.Auto) + { + // The scenario databases are reached over a public address with no TLS, so Auto should have + // derived external-ip: an address, not a hostname, and not null. + var named = moving[0].NewEndPoint; + Assert.NotNull(named); + Assert.IsType(named); + } + + // The one thing worth asserting either way: whatever the server sent, we understood the frame. + Assert.All(moving, push => Assert.Equal(MaintenanceNotificationType.Moving, push.NotificationType)); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs new file mode 100644 index 000000000..a5c6acbee --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/MovingHandoffScenarioTests.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// D6 against a real deployment: do we get off the connection before the server takes it away? +/// +/// +/// The one thing the in-process harness structurally cannot test. The handoff's hostname branch needs two +/// things a fake cannot supply - an endpoint that is a *name*, and a real socket whose remote address tells us +/// where we currently are - and its decision turns on DNS changing underneath us. +/// +/// The discriminator is the failure type of the disconnect. ConnectionDisposed means *we* replaced the +/// connection; SocketClosed means the server did. Both outcomes are legitimate, because DNS is not +/// guaranteed to win the race - measured on one cluster at +18.7s against a socket closing at +15.7s - so this +/// records which happened rather than demanding the good one. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "moving-handoff")] +public class MovingHandoffScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("conn_drop", "endpoint_rebind", MaintenanceEndpointType.ServerDefault)] + [InlineData("data_movement_conn_drop", "maintenance_mode", MaintenanceEndpointType.ServerDefault)] + [InlineData("conn_drop", "endpoint_rebind", MaintenanceEndpointType.ExternalFqdn)] + public async Task HandoffBeatsTheServerToTheClose(string effect, string trigger, MaintenanceEndpointType endpointType) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", effect, trigger, log.WriteLine, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var clock = Stopwatch.StartNew(); + var timeline = new List(); + void Note(string what) + { + var entry = $" +{clock.Elapsed.TotalSeconds,6:0.0}s {what}"; + lock (timeline) timeline.Add(entry); + log.WriteLine(entry); + } + + var config = database.GetClientConfig(fixture.Environment); + config.MaintenanceMovingEndpointType = endpointType; + log.WriteLine($"moving-endpoint-type: {endpointType}"); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var muxer = (IInternalConnectionMultiplexer)conn; + var endpoint = muxer.GetServerEndPoint(conn.GetEndPoints()[0]); + + // The precondition the fake could not meet: a name to resolve, and a socket that says where we are. + Assert.IsType(conn.GetEndPoints()[0]); + log.WriteLine($"dialled {conn.GetEndPoints()[0]} (a name, so the probe can engage)"); + + var failures = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (failures) failures.Add(e.FailureType); + Note($"disconnect: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => Note("reconnected"); + var movingSequences = new HashSet(); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is not PushMaintenanceEvent push) return; + Note($"{push.NotificationType} seq={push.SequenceId} time={push.Time?.TotalSeconds.ToString() ?? "-"}"); + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + lock (movingSequences) movingSequences.Add(push.SequenceId); + } + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + Note("injector reports the scenario finished"); + + // long enough to cover the whole announced window plus the observed overshoot + var deadline = clock.Elapsed + TimeSpan.FromSeconds(75); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + Note($"ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + Note($"handoff outcome: {endpoint.LastHandoffOutcome ?? "(none recorded)"}; recycles={endpoint.HandoffRecycles}"); + + lock (failures) + { + var order = string.Join(" -> ", failures); + log.WriteLine($" disconnect sequence: {(order.Length == 0 ? "(none)" : order)}"); + + // Note what is *not* asserted: that a ConnectionDisposed appears here. Our own recycle does not + // raise ConnectionFailed - disposal is not reported as a failure - so from the outside a handoff is + // invisible, which is worth knowing in its own right and is why HandoffRecycles exists. + } + + // Exactly one handoff per distinct notification - not one overall, which is what this asserted until a + // live run proved the scenario can announce twice. The `maintenance_mode` trigger reports + // `automatically_clean_mm: false`, so the node stays in maintenance mode after the effect lands and + // coming back out of it moves the shard again: MOVING seq=2 at +14.9s and seq=3 at +97.0s, one recycle + // each, and the client was right both times. + // + // What the count still catches is the feedback loop this test found on its first live run: a server + // re-sends MOVING to a connection that opts in while the window is still open, and since the handoff + // replaces the connection, acting on the repeat produces another one - twelve recycles from *one* + // event. A replay repeats the sequence number, so keying on the sequence keeps that sharp while + // letting a genuine second event through. + int announced; + lock (movingSequences) announced = movingSequences.Count; + Assert.True(announced > 0, "no MOVING was announced, so this test would prove nothing"); + Assert.Equal(announced, endpoint.HandoffRecycles); + Assert.NotNull(endpoint.LastHandoffOutcome); + + // Which outcome depends on whether the server named a replacement, and it only does that when we asked + // for an endpoint type: MoveTo when it did, Recycle when we had to find the new address via DNS. + var expectedOutcome = endpointType == MaintenanceEndpointType.ServerDefault ? "Recycle" : "MoveTo"; + Assert.Contains(expectedOutcome, endpoint.LastHandoffOutcome); + + if (endpointType != MaintenanceEndpointType.ServerDefault) + { + // The payoff, and the reason to ask for an endpoint type at all: with somewhere named to go we move + // immediately instead of waiting on DNS, so the server never has to close the connection out from + // under us. Measured before this worked: the handoff happened but landed back on the node being + // retired, and a SocketClosed followed ~15s later. + lock (failures) + { + Assert.DoesNotContain(ConnectionFailureType.SocketClosed, failures); + } + } + + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should be serving commands after the handoff"); + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs new file mode 100644 index 000000000..3deeadc17 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/NodeRemovalScenarioTests.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// An OSS cluster API database, so the client holds one endpoint per node and can retire one. +public sealed class OssClusterLifecycleFixture() : FaultInjectorFixture(DatabaseShape.OssClusterApi); + +/// +/// A node removed from the cluster: endpoint retirement measured against a real deployment rather than a fake. +/// +/// +/// The one destructive scenario that exercises something this feature built. Pruning exists because the +/// endpoint collection used to be add-only: a node that left the cluster was dialled forever, which is half of +/// the 37-hour field failure. Every test of it so far has been against the in-process server, where "the node +/// left" is a method call; here the node genuinely leaves, the topology genuinely changes, and the endpoint has +/// to be let go without dropping the caller's work. +/// +/// Needs the OSS cluster API shape. A proxied standalone database is reached through one hostname however many +/// nodes are behind it, so there is no per-node endpoint to retire and the test would be vacuous. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "destructive")] +public class NodeRemovalScenarioTests(OssClusterLifecycleFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string EnableVariable = "SER_FI_DESTRUCTIVE"; + + private static bool Enabled => + string.Equals(Environment.GetEnvironmentVariable(EnableVariable), "true", StringComparison.OrdinalIgnoreCase); + + [Fact] + public async Task RemovingANodeRetiresItsEndpoint() + { + if (!Enabled) Assert.Skip($"set {EnableVariable}=true to run the destructive scenarios; they damage the cluster"); + + fixture.RequireAvailable(); + var database = fixture.Database; + Assert.NotNull(database); + var cancellationToken = TestContext.Current.CancellationToken; + + var nodes = await ClusterNodes.ListAsync(fixture.Injector, database.BdbId, cancellationToken); + log.WriteLine($"cluster nodes: {string.Join(", ", nodes.Select(n => $"{n.Id}={n.Role}@{n.ExternalAddress}"))}"); + if (nodes.Count < 3) Assert.Skip($"only {nodes.Count} node(s); removing one needs somewhere for its shards to go"); + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + var db = conn.GetDatabase(); + const string Key = "fi-node-remove"; + await db.StringSetAsync(Key, "before"); + + conn.ConnectionFailed += (_, e) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s failed: {e.FailureType}"); + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s restored"); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId}"); + } + }; + + var before = conn.GetEndPoints(); + log.WriteLine($"endpoints before: {string.Join(", ", before.Select(e => e.ToString()))}"); + + // The node whose endpoint we hold *and* which is not serving our own connection, so the removal is + // visible as a retirement rather than as a reconnect. If we cannot tell them apart, any node we hold + // an endpoint for will do - the retirement is the assertion either way. + var serving = await ClusterNodes.FindServingAsync(fixture.Injector, database.BdbId, database.Host, cancellationToken); + + // not the node serving us, and not node 1: cluster management lives there on a default install, and + // taking it out takes the fault injector's own access with it + var candidate = nodes.FirstOrDefault(n => n.Id != serving?.Id && n.Id != 1 && n.Role != "master"); + if (candidate is null) Assert.Skip("no node that is safe to remove and observable from here"); + + log.WriteLine($"removing node {candidate.Id} ({candidate.ExternalAddress}); we are served by node {serving?.Id.ToString() ?? "(unknown)"}"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "node_remove", + new Dictionary { ["node_id"] = candidate.Id.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not remove node {candidate.Id}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports node_remove finished"); + + // Retirement needs several consecutive topology passes that do not list the node, and those are driven + // by the config-check interval, so this is tens of seconds rather than immediate by design. + var retired = await Poll.UntilAsync( + () => !conn.GetEndPoints().Any(e => e.ToString()!.Contains(candidate.ExternalAddress, StringComparison.OrdinalIgnoreCase)), + timeoutMilliseconds: 180_000, + pollMilliseconds: 2000); + + log.WriteLine($"endpoints after: {string.Join(", ", conn.GetEndPoints().Select(e => e.ToString()))}"); + + // The caller's work is the part that must not suffer, whatever the endpoint collection does. + Assert.Equal("before", await db.StringGetAsync(Key)); + await db.StringSetAsync(Key, "after"); + Assert.Equal("after", await db.StringGetAsync(Key)); + + if (!retired) + { + // Reported rather than asserted: the endpoint set a proxied cluster advertises does not have to + // name every node, so "the address never appeared in our endpoints" is a legitimate outcome and + // not a pruning failure. The log above says which it was. + log.WriteLine( + $" note: {candidate.ExternalAddress} was not present in our endpoint set, or was not retired within the bound; " + + "traffic was unaffected either way"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs new file mode 100644 index 000000000..0d93b2db2 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Poll.cs @@ -0,0 +1,21 @@ +using System; +using System.Threading.Tasks; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Waits for a condition that a real deployment reaches on its own schedule. +/// +internal static class Poll +{ + public static async Task UntilAsync(Func condition, int timeoutMilliseconds = 10_000, int pollMilliseconds = 250) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMilliseconds); + while (true) + { + if (condition()) return true; + if (DateTime.UtcNow > deadline) return false; + await Task.Delay(pollMilliseconds); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs new file mode 100644 index 000000000..dd2135c74 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/ProxyAndFailoverScenarioTests.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// A replicated database, provisioned by us - the only way to reach the failover family. +/// +/// +/// The environment templates create databases with replication: false, and a failover needs a replica to +/// promote, so this is the one case where create_database earns its keep rather than being a +/// less-convenient alternative to the scenario setup legs. +/// +public sealed class ReplicatedDatabaseFixture() + : FaultInjectorFixture(new DatabaseShape("replicated", ProxyPolicy: "single", Replication: true, ShardCount: 2)); + +/// +/// The proxy and failover families: FAILING_OVER/FAILED_OVER, and a proxy restarting underneath us. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "failover")] +public class FailoverScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Fact] + public async Task FailoverIsAnnouncedAndSurvived() + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + log.WriteLine($"provisioned {database}"); + + var clock = Stopwatch.StartNew(); + var events = new List(); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + } + }; + + var db = conn.GetDatabase(); + await db.StringSetAsync("fi-failover", "before"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "failover", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + // The failover action's parameters are untyped in the schema, so a rejection here is a harness + // problem rather than a client finding; say so plainly instead of reporting it as a product failure. + Assert.Skip($"the injector would not run 'failover' against bdb {database.BdbId}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports the failover finished"); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + try + { + await db.PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s): {string.Join(", ", events.Select(e => e.NotificationType))}"); + + // A failover is the one event whose *pair* we have never observed end to end from a client: Marc + // captured the frames by hand, but nothing has watched SE.Redis receive them. + Assert.NotEmpty(events); + Assert.Contains(events, e => e.NotificationType is MaintenanceNotificationType.FailingOver + or MaintenanceNotificationType.FailedOver + or MaintenanceNotificationType.Migrating // a failover on a proxied database moves shards too + or MaintenanceNotificationType.Migrated + or MaintenanceNotificationType.Moving); + } + + // the data survived, which is the point of replication + Assert.Equal("before", await db.StringGetAsync("fi-failover")); + } +} + +/// +/// The proxy process restarting: no topology change, no notification, just the socket going away. +/// +/// +/// Included because it is the one disruption that is *purely* a connection event - nothing moves, nothing is +/// announced, and recovery is entirely the client's ordinary reconnect path. A useful control: if this fails, +/// the failures in the announced scenarios are not about notifications at all. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "proxy-restart")] +public class ProxyRestartScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("standalone")] + [InlineData("cluster")] + public async Task ProxyRestartIsSurvived(string key) + { + var database = fixture.Require(key); + var cancellationToken = TestContext.Current.CancellationToken; + + var clock = Stopwatch.StartNew(); + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var db = conn.GetDatabase(); + await db.StringSetAsync($"fi-dmc-{key}", "before"); + + int drops = 0; + conn.ConnectionFailed += (_, e) => + { + drops++; + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s connection failed: {e.FailureType}"); + }; + conn.ConnectionRestored += (_, _) => log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s connection restored"); + + clock.Restart(); + // the one action with a typed parameter object in the injector's schema: RestartDmcParams { bdb_id } + await fixture.Injector.RunActionAsync( + "dmc_restart", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s DMC restart reported finished"); + + // Recovery is asserted by polling rather than by waiting a fixed time, because a proxy restart is quick + // and the interesting failure is "never comes back", not "takes a moment". + Assert.True( + await Poll.UntilAsync(() => TryRead(db, $"fi-dmc-{key}"), timeoutMilliseconds: 60_000), + "the client should recover from the proxy restarting"); + + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s recovered after {drops} drop(s)"); + Assert.Equal("before", await db.StringGetAsync($"fi-dmc-{key}")); + } + + private static bool TryRead(IDatabase db, string key) + { + try + { + return db.StringGet(key) == "before"; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md new file mode 100644 index 000000000..a3a6f2356 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -0,0 +1,128 @@ +# StackExchange.Redis.FaultInjector.Tests + +Scenario tests that drive a **real Redis Enterprise deployment** through the fault injector, and watch how +SE.Redis reacts. These are the tier that can observe what no in-process fake can: real DNS, real TLS identity, +real timing. + +## Running them + +```bash +cd # holds docker-compose.yml, env_output.json, ... +docker compose up -d # 10-15 minutes for the cluster to come up + +export SER_FI_CONFIG_DIR=$PWD # or FI_CONSOLE_CONFIG_DIR, which is also honoured +export E2E_SCENARIO_TESTS=true # explicit opt-in: these create and delete databases +dotnet test tests/StackExchange.Redis.FaultInjector.Tests +``` + +**Point it at the *current* environment.** A stale directory passes every check we make: the credentials are +only checked for being well-formed, and node discovery and database provisioning both go through the injector, +which is configured separately - so the only thing that fails is the template databases, which belong to a +cluster that no longer exists. If `endpoints.json` names a different cluster than the one the injector is +driving, that directory is not the one you want. + +One path is the whole configuration. That directory is the one mounted into the injector as `/app/config`, so +it already holds the cluster credentials (`env_output.json`), the CA certificate, and the compose file; nothing +has to be hand-carried into the test run. `FAULT_INJECTION_API_URL` overrides the injector URL +(default `http://127.0.0.1:20324`). + +## The one test that is opt-in even here + +`RetentionAgeScenarioTests` measures how long the server retains a completion for replay to a +newly-opted-in connection - the last unmeasured property of the catch-up channel. It fires one failover and +then probes on a ladder (1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 180, 240 minutes), so it runs for as long as +you let it and skips unless you ask for it: + +```bash +export SER_FI_RETENTION_AGE_MINUTES=180 # trims the ladder; absent means skip +export SER_FI_RETENTION_AGE_LOG=/tmp/age.log # optional; defaults under the temp directory +``` + +Two details that are not incidental: + +- **Progress is written to a file, flushed per line.** `ITestOutputHelper` is buffered until the test ends, so + over three hours a run in progress and a run that has wedged look identical through the normal channel. +- **Two probes per rung.** If the first sees the replay and the second does not, the server clears the retained + item on delivery - in which case later rungs are measuring an empty channel rather than an expired one. That + confound is invisible with one probe per rung and looks exactly like an early expiry. (Measured 2026-09-02: + both probes see it, so retention is not consumed on delivery.) +- A probe that cannot connect is recorded as **inconclusive, not as a miss**: the run outlives its cluster's + lease easily, and counting a dead environment as "no replay" would report an expiry at whatever minute the + cluster went away. + +## The destructive scenarios, behind a second gate + +`DestructiveScenarioTests` breaks things rather than moving them - a shard, a node, or a proxy is killed - so +it needs its own opt-in on top of the tier's: + +```bash +export SER_FI_DESTRUCTIVE=true # absent means skip +export SER_FI_NODE_TO_KILL=2 # optional; which node node_failure kills (default 2, never 1) +``` + +`E2E_SCENARIO_TESTS` says "you may create and delete databases"; it does not say "you may kill nodes", and a +cluster that has to be re-provisioned is 10-15 minutes of somebody's afternoon. Run these at the end of a +cluster's life, watching. + +Two things measured on 2026-09-03 that change how you read a green run: + +- **The scope differs per action**, and the injector's schema does not say so. Learned by being told: + + | action | parameter | + |---|---| + | `shard_failure`, `proxy_failure` | `bdb_id` | + | `node_failure`, `node_remove` | `node_id` | + | `cluster_failure` | `node_ids` (a list) | + | `execute_rladmin_command` | `bdb_id` *and* `rladmin_command` | + +- **`SER_FI_CLUSTER_FAILURE=true` is a separate gate, and it ends the deployment.** `cluster_failure` stops + the nodes you name and restores nothing, so naming them all leaves the cluster down for good - `rladmin` + stops answering and the environment needs re-provisioning. Run it last or not at all. +- **Node-scoped actions need the *right* node.** `ClusterNodes.FindServingAsync` resolves the database + hostname and matches it against `rladmin status nodes`, because killing an arbitrary node usually proves + nothing: the deployment absorbs it and the client never notices. Node discovery goes through the injector + rather than the cluster's REST API on 9443, which is not reachable from outside the deployment's network. +- **Only `proxy_failure` was visible to the client** (one `SocketClosed`, restored ~8s later). `shard_failure` + on a replicated database and `node_failure` against a node we were not connected through both produced zero + drops - the deployment absorbed them. That is worth knowing and is *not* coverage of our recovery path, so + the test says so in its output when it sees no drops. Node 1 is avoided by default because cluster + management usually lives there, and killing it takes the fault injector's own access with it. + +## Three states, deliberately distinct + +| state | behaviour | +|---|---| +| no `SER_FI_CONFIG_DIR` | every test **skips** - the ordinary case, including `build.ps1`'s full traversal | +| directory set, no `E2E_SCENARIO_TESTS=true` | every test **skips** - nobody should create databases by accident | +| configured and enabled, but broken | every test **fails** | + +The third row is the important one. A suite that skips when the environment is broken reports success for tests +that never ran, and it will be trusted at exactly the wrong moment. + +## Databases are created by the tests, not by you + +Each *shape* (`DatabaseShape`) is a fixture shared by the classes that need it, because creating a database on a +real cluster is slow. The shapes exist because they change client behaviour rather than for coverage's sake: the +number of A records a hostname carries follows proxy placement, and the handoff takes a different branch +depending on whether a live sibling address exists. + +Every database is named `sertest--`. Cleanup is per fixture and unconditional; a sweep at startup +removes leaks from runs that were killed, matching on the `sertest-` prefix and nothing else, so it can never +touch a database created by hand. + +## TLS + +Certificates are self-signed per environment, so tests call `ConfigurationOptions.TrustIssuer(caPath)` with the +CA found in the config directory. If the CA is missing, TLS tests **fail** rather than disabling validation - a +TLS test that quietly stops checking identity reports success for the one thing it exists to catch. + +## Traits + +`tier=fault-injector` on everything, so the whole tier can be excluded in one filter; `scenario=` for +subsets. + +## Unverified + +The `create_database` parameter names in `DatabaseShape.ToCreateParameters` are the injector's wire schema, +which is documented only as prose. They are gathered in one place so a real run can correct them; go-redis's +`DatabaseConfig` is the closest reference implementation. Treat them as unconfirmed until a run accepts them. diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs new file mode 100644 index 000000000..a5f37b8b6 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/RealDeploymentSmokeTests.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Does the client work against a real deployment at all: connect, negotiate RESP3, opt in, run a command. +/// +/// +/// The floor for everything else. Worth having as its own class because when a scenario test fails, the first +/// question is whether the deployment was reachable in the first place, and a green smoke test answers it +/// without reading logs. +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "smoke")] +public class RealDeploymentSmokeTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData("standalone")] + [InlineData("cluster")] + public async Task OptInIsAcceptedAndTheConnectionWorks(string key) + { + var database = fixture.Require(key); + log.WriteLine($"{database}"); + log.WriteLine($" advertised addresses: {string.Join(", ", database.Addresses)}"); + log.WriteLine($" endpoint type: {database.EndpointType ?? "(unset)"}"); + + // Enabled refuses the connection if the server will not give us notifications, so reaching the + // assertions at all is the opt-in having been accepted - there is nothing weaker to check. A stub +OK + // would pass this, which is what the scenario tests are for. + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + Assert.True(conn.IsConnected); + + var server = conn.GetServer(conn.GetEndPoints()[0]); + log.WriteLine($" connected: {server.Version}, {server.ServerType}, protocol {server.Protocol}"); + Assert.Equal(RedisProtocol.Resp3, server.Protocol); + + var rtt = await conn.GetDatabase().PingAsync(); + log.WriteLine($" ping: {rtt.TotalMilliseconds:0.0}ms"); + Assert.True(rtt > TimeSpan.Zero); + + // and the feature is actually live on this connection, not merely requested + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "maintenance notifications should be active"); + } + + [Fact] + public async Task AdvertisedAddressCountMatchesTheProxyPolicy() + { + // Not a client assertion: a check that the environment is the shape the handoff tests assume. The count + // follows proxy *placement* rather than the policy name, so this records what this deployment actually + // is - and a handoff test that expects a sibling to step to needs more than one. + var standalone = fixture.Require("standalone"); + log.WriteLine($"{standalone.Key}: policy {standalone.ProxyPolicy}, {standalone.AdvertisedAddressCount} address(es)"); + + Assert.NotEmpty(standalone.Addresses); + if (standalone.ProxyPolicy is "single") + { + Assert.Equal(1, standalone.AdvertisedAddressCount); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs new file mode 100644 index 000000000..1d728b129 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/RetentionAgeScenarioTests.cs @@ -0,0 +1,359 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.Logging; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// How long a retained completion stays retained - the last unmeasured property of the catch-up channel. +/// +/// +/// What is already known, from captures: a connection that opts in *after* a shard-scoped event gets the +/// event's **completion** replayed (MIGRATED, FAILED_OVER; never a starter, never +/// MOVING), delivered within ~17ms of the opt-in being accepted, most-recent-replaces. What is not +/// known is whether that retention ever ages out. It matters because a client is entitled to act on what it +/// receives: a completion replayed hours later would open a relaxed window for an event that finished long +/// ago, which is harmless-but-wasteful for us and would be worth an age guard if the server does not have one. +/// +/// Written as a measurement rather than a pass/fail: the schedule is a ladder, every probe is logged, and the +/// only hard assertions are the ones that say the measurement itself is sound. Opt in with +/// SER_FI_RETENTION_AGE_MINUTES=<minutes>; without it this skips, because it fires one failover +/// and then spends the rest of its time waiting, which has no place in an ordinary run of the tier. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "retention-age")] +public class RetentionAgeScenarioTests(ReplicatedDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + private const string HorizonVariable = "SER_FI_RETENTION_AGE_MINUTES"; + private const string ProgressVariable = "SER_FI_RETENTION_AGE_LOG"; + + /// + /// Progress is written to a file as it happens, as well as to the test output. + /// + /// + /// is buffered until the test finishes, and this test runs for hours - so + /// through the only channel a test normally has, a run in progress and a run that has wedged look + /// identical. The file is flushed per line, so the ladder can be read while it is still being climbed. + /// + private static string ProgressPath => + Environment.GetEnvironmentVariable(ProgressVariable) is { Length: > 0 } configured + ? configured + : Path.Combine(Path.GetTempPath(), "ser-retention-age.log"); + + /// Probe ages, in minutes since the completion; trimmed to whatever horizon was asked for. + /// + /// Dense early and sparse later: an expiry at 30 seconds and an expiry at two hours are both plausible, and + /// a geometric ladder pins either to within a factor of two without spending the whole cluster lease. + /// + private static readonly int[] LadderMinutes = [1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 180, 240]; + + [Fact] + public async Task HowLongIsACompletionRetained() + { + fixture.RequireAvailable(); + var horizon = ReadHorizon(); + if (horizon is null) + { + Assert.Skip( + $"set {HorizonVariable}= to run this; it fires one failover and then probes for that " + + "long, so it is opt-in even within this tier"); + } + + var cancellationToken = TestContext.Current.CancellationToken; + var database = fixture.Database; + Assert.NotNull(database); + + using var progress = new StreamWriter(ProgressPath, append: true) { AutoFlush = true }; + _progress = progress; + Note($"--- retention age, horizon {horizon} minutes, started {DateTime.UtcNow:u} ---"); + Note($"provisioned {database}"); + + // The witness stays connected for the whole run, for one reason: retention is most-recent-replaces, so + // any *further* event on this database resets the age we are measuring. If one arrives, the ladder from + // that point on is measuring the new event, and the log has to show that rather than hide it. + var clock = Stopwatch.StartNew(); + var witnessed = new List<(TimeSpan At, PushMaintenanceEvent Push)>(); + await using var witness = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig()); + witness.ServerMaintenanceEvent += (_, e) => + { + if (e is not PushMaintenanceEvent push) return; + lock (witnessed) witnessed.Add((clock.Elapsed, push)); + Note($" witness +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + }; + + await witness.GetDatabase().StringSetAsync("fi-retention-age", "before"); + + clock.Restart(); + try + { + await fixture.Injector.RunActionAsync( + "failover", + new Dictionary { ["bdb_id"] = database.BdbId.ToString() }, + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + Assert.Skip($"the injector would not run 'failover' against bdb {database.BdbId}: {ScenarioSupport.Summarize(ex.Message)}"); + } + + Note($" +{clock.Elapsed.TotalSeconds,6:0.0}s injector reports the failover finished"); + + // A completion is what gets retained, so the clock we care about starts when one arrives - not when the + // scenario was fired, and not when the injector called it done. + var completed = await Poll.UntilAsync( + () => + { + lock (witnessed) return witnessed.Any(w => IsCompletion(w.Push.NotificationType)); + }, + timeoutMilliseconds: 120_000); + + if (!completed) + { + lock (witnessed) + { + var seen = witnessed.Count == 0 ? "nothing" : string.Join(", ", witnessed.Select(w => w.Push.NotificationType)); + Assert.Skip($"no completion was announced within 120s (saw {seen}), so there is nothing whose retention could be measured"); + } + } + + TimeSpan completionAt; + long completionSequence; + MaintenanceNotificationType completionType; + lock (witnessed) + { + var completion = witnessed.First(w => IsCompletion(w.Push.NotificationType)); + completionAt = completion.At; + completionSequence = completion.Push.SequenceId; + completionType = completion.Push.NotificationType; + } + + Note($"completion to measure: {completionType} seq={completionSequence} at +{completionAt.TotalSeconds:0.0}s"); + + var results = new List(); + foreach (var minutes in LadderMinutes.Where(m => m <= horizon)) + { + var due = completionAt + TimeSpan.FromMinutes(minutes); + var wait = due - clock.Elapsed; + if (wait > TimeSpan.Zero) await Task.Delay(wait, cancellationToken); + + // Two connections per rung, back to back. If the first sees the replay and the second does not, the + // server clears the retained item once it has been delivered - in which case every later rung is + // measuring an empty channel rather than an expired one, and the ladder means nothing. That + // confound is invisible with one connection per rung, and it would look exactly like an early + // expiry. + var first = await ProbeAsync(database, minutes, "a", cancellationToken); + var second = await ProbeAsync(database, minutes, "b", cancellationToken); + results.Add(first); + results.Add(second); + + if (first.Replayed && !second.Replayed) + { + Note( + $" !! at {minutes}m the first probe saw the replay and the second did not: retention looks " + + "consumed on delivery, so rungs beyond this one cannot be read as ages"); + } + } + + Note(string.Empty); + Note("age probe replayed type notes"); + foreach (var probe in results) + { + var verdict = probe.Conclusive ? (probe.Replayed ? "yes" : "no") : "n/a"; + Note($"{probe.Minutes,4}m {probe.Label,-5} {verdict,-8} {probe.Type,-12} {probe.Notes}"); + } + + lock (witnessed) + { + var later = witnessed.Where(w => w.At > completionAt).ToList(); + if (later.Count != 0) + { + Note( + " !! further events arrived after the completion, so the retained item was replaced and the " + + $"ages above are relative to the wrong event: {string.Join(", ", later.Select(w => $"{w.Push.NotificationType}@+{w.At.TotalSeconds:0}s"))}"); + } + } + + // The measurement is the output; these two assertions exist so that a run which proves nothing says so + // instead of being read as "retention expires immediately". + Assert.NotEmpty(results); + Assert.True(results[0].Conclusive, $"the first probe could not connect: {results[0].Notes}"); + Assert.True( + results[0].Replayed, + $"the first probe ({results[0].Minutes}m after a {completionType}) saw no replay at all, so this run " + + "measured nothing - either retention is shorter than the first rung, or the opt-in is not being honoured"); + + // Once it stops being replayed it must stay stopped. An age guard is monotone; a completion that + // reappears after a gap would mean something much stranger than expiry, and is worth failing on. + var byRung = results.Where(r => r is { Label: "a", Conclusive: true }).ToList(); + if (byRung.Count == 0) + { + Assert.Fail("every probe failed to connect, so nothing was measured"); + } + + var firstMiss = byRung.FindIndex(r => !r.Replayed); + if (firstMiss >= 0) + { + var after = byRung.Skip(firstMiss).Where(r => r.Replayed).ToList(); + Assert.True( + after.Count == 0, + $"the replay stopped at {byRung[firstMiss].Minutes}m and then came back at " + + $"{string.Join(", ", after.Select(r => r.Minutes + "m"))}, which no expiry rule explains"); + Note($"=> retention lapsed between {(firstMiss == 0 ? 0 : byRung[firstMiss - 1].Minutes)}m and {byRung[firstMiss].Minutes}m"); + } + else + { + Note($"=> still replayed at {byRung[^1].Minutes}m: retention outlasts the horizon asked for"); + } + } + + /// + /// One fresh connection, and what the server told it on the way in. + /// + /// + /// The observable is the client's own log, not the ServerMaintenanceEvent, and that is not a + /// convenience: a retained completion arrives within ~17ms of the opt-in being accepted, which is *inside* + /// ConnectAsync, so a handler attached after connecting has already missed it. A logger can be + /// attached through the configuration beforehand. + /// + /// It used to read the endpoint's relaxed window instead, which was simpler and is no longer true: since a + /// catch-up completion is history rather than news, it deliberately opens no window - a change this + /// measurement is what prompted. Reading the window would now report "not replayed" at every rung, which + /// would look exactly like an expiry at the first one. + /// + /// + private async Task ProbeAsync(ProvisionedDatabase database, int minutes, string label, CancellationToken cancellationToken) + { + try + { + var options = database.GetClientConfig(); + var notifications = new NotificationLog(); + options.LoggerFactory = notifications; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(options); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + var received = notifications.Received; + var relaxed = received.Count != 0; + var type = received.Count == 0 ? MaintenanceNotificationType.None : TypeOf(received[0]); + if (endpoint.IsMaintenanceRelaxed) + { + // not expected on a fresh connection any more; if it happens, something arrived *live* while + // we were connecting, and the rung is measuring that instead + Note($" probe {minutes}m/{label}: note - the window is open ({endpoint.ActiveMaintenanceType}), so a live event may be in play"); + } + + // a live event arriving *during* the probe would also relax the window, so record anything that + // shows up while we are here; the witness sees it too, and the two together tell them apart + var live = new List(); + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) lock (live) live.Add(push.NotificationType); + }; + await conn.GetDatabase().PingAsync(); + await Task.Delay(2000, cancellationToken); + + string notes; + lock (live) + { + notes = live.Count == 0 ? string.Empty : $"also received live: {string.Join(", ", live)}"; + } + + if (received.Count != 0) + { + notes = string.IsNullOrEmpty(notes) ? received[0] : $"{received[0]}; {notes}"; + } + + Note($" probe {minutes}m/{label}: relaxed={relaxed} type={type} {notes}"); + return new Probe(minutes, label, relaxed, type, notes, Conclusive: true); + } + catch (Exception ex) + { + // Inconclusive, emphatically not "no replay": this runs for hours against a cluster with a lease on + // it, and a probe that cannot connect at all says nothing about retention. Counting it as a miss + // would report an expiry at whatever minute the environment went away. + Note($" probe {minutes}m/{label}: failed to connect: {ex.GetType().Name}: {ex.Message}"); + return new Probe(minutes, label, false, MaintenanceNotificationType.None, $"connect failed: {ex.GetType().Name}", Conclusive: false); + } + } + + private StreamWriter? _progress; + + private void Note(string message) + { + log.WriteLine(message); + _progress?.WriteLine(message.Length == 0 ? message : $"{DateTime.UtcNow:HH:mm:ss} {message}"); + } + + /// Reads the notification type back out of the log line, which is the only place it is stated. + private static MaintenanceNotificationType TypeOf(string logLine) + { + foreach (var candidate in Enum.GetValues()) + { + if (candidate != MaintenanceNotificationType.None + && logLine.Contains(candidate.ToString(), StringComparison.Ordinal)) + { + return candidate; + } + } + + return MaintenanceNotificationType.None; + } + + /// Captures the client's own "maintenance notification" lines, attached before connecting. + private sealed class NotificationLog : ILoggerFactory, ILogger + { + private readonly List _received = []; + + public List Received + { + get { lock (_received) return [.. _received]; } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + if (message.Contains("Maintenance notification:", StringComparison.Ordinal)) + { + lock (_received) _received.Add(message); + } + } + + public void Dispose() { } + } + + private static bool IsCompletion(MaintenanceNotificationType type) + => type is MaintenanceNotificationType.Migrated or MaintenanceNotificationType.FailedOver; + + private static int? ReadHorizon() + { + var raw = Environment.GetEnvironmentVariable(HorizonVariable); + return int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var minutes) && minutes > 0 + ? minutes + : null; + } + + private readonly record struct Probe( + int Minutes, + string Label, + bool Replayed, + MaintenanceNotificationType Type, + string Notes, + bool Conclusive); +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj new file mode 100644 index 000000000..052d85e11 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + Exe + StackExchange.Redis.FaultInjector.Tests + true + true + full + enable + true + + + + + + + + + + + + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs new file mode 100644 index 000000000..7d0e61153 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TlsScenarioTests.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.IO; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// The same feature over TLS, against certificates a real cluster generated. +/// +/// +/// A gap that has been open since the identity work: nothing had exercised a real TLS deployment, only the +/// in-process harness with a certificate it made itself. That matters because the interesting failures are +/// about *identity* - which name the certificate carries, and whether the endpoint we are told to use is +/// covered by it - and a fake that issues its own certificate cannot be wrong about that in the way a real +/// deployment can. +/// +/// Certificates here are self-signed per environment, so the client pins the issuer with +/// . Note what is deliberately *not* done: validation is +/// never disabled. A TLS test that stops checking identity reports success for precisely the thing it exists to +/// catch, so a missing CA fails the run instead. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "tls")] +public class TlsScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + [Theory] + [InlineData(1, "single_tls", false)] + [InlineData(2, "mtls", true)] + public async Task NotificationsArriveOverTlsAndIdentityIsVerified(int variantIndex, string expectedConfig, bool expectClientCertificate) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + if (fixture.Environment.CertificateAuthorityPath is null) + { + // a skip rather than a failure: whether the environment generated certificates is a provisioning + // choice, and running this without them would prove nothing + Assert.Skip($"no CA certificate in {fixture.Environment.ConfigDirectory.FullName}; provision with TLS to run this"); + } + + log.WriteLine($"trusting issuer {fixture.Environment.CertificateAuthorityPath}"); + CertificateSanity.RequireCertificatesMatchThisCluster(fixture.Environment, log.WriteLine); + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, + "topology-change-standalone", + "conn_drop", + "endpoint_rebind", + log.WriteLine, + // include_tls does not *request* TLS, it widens the list of variants the setup may choose from; + // variant_index is what picks one. Confirmed by asking the discovery endpoint: with no flags a + // trigger offers one variant ("single"), with include_tls two ("single", "single_tls"), and with + // include_mtls a third ("mtls"). Passing include_tls alone provisions variant 0 and yields a + // plaintext database, which is how this test first came to skip itself. + // include_tls / include_mtls widen the variant list; variant_index picks one. With no flags a + // trigger offers just "single"; include_tls adds "single_tls"; include_mtls adds "mtls", which is + // the same TLS database plus enforce_client_authentication. + extra: new Dictionary + { + ["include_tls"] = "true", + ["include_mtls"] = expectClientCertificate ? "true" : null, + ["variant_index"] = variantIndex.ToString(), + }, + cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + log.WriteLine($"provisioned {database} (tls={database.Tls})"); + + if (!database.Tls) + { + Assert.Skip("the injector provisioned a plaintext database despite include_tls=true; nothing to test here"); + } + + Assert.Equal(expectedConfig, database.ProxyPolicy); // the setup reports which variant it built + if (expectClientCertificate) + { + // A database with enforce_client_authentication rejects a connection that offers no certificate, so + // the material has to be there: this is the client identity, issued by the injector's own + // intermediate CA, and a different trust root from the one validating the server. + Assert.NotNull(database.Mtls); + log.WriteLine($"presenting client certificate {database.Mtls.ClientCertificatePath}"); + Assert.True(File.Exists(database.Mtls.ClientCertificatePath), $"missing {database.Mtls.ClientCertificatePath}"); + Assert.True(File.Exists(database.Mtls.ClientKeyPath), $"missing {database.Mtls.ClientKeyPath}"); + } + else + { + Assert.Null(database.Mtls); + } + + var clock = Stopwatch.StartNew(); + var events = new List(); + + var config = database.GetClientConfig(fixture.Environment); + + // AbortOnConnectFail=true *for this test only*: everywhere else tolerating a slow start is right, but a + // TLS failure has to surface its reason. With it false, a certificate problem is indistinguishable from + // a slow cluster - ConnectAsync succeeds, IsConnected is false, and the cause is gone. + config.AbortOnConnectFail = true; + + // If the certificate does not cover the name we dialled, this throws - which is the point: the + // assertion that identity was verified is the connect succeeding with validation on. The log goes to + // test output so a handshake failure says *which* check failed. + var connectLog = new StringWriter(); + ConnectionMultiplexer conn; + try + { + conn = await ConnectionMultiplexer.ConnectAsync(config, connectLog); + } + catch (Exception ex) + { + log.WriteLine(connectLog.ToString()); + log.WriteLine($"TLS connect failed: {ex.GetType().Name}: {ex.Message}"); + throw; + } + + await using (conn) + { + Assert.True(conn.IsConnected); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "the opt-in should be live over TLS too"); + log.WriteLine($"connected over TLS; opt-in active; ping {(await conn.GetDatabase().PingAsync()).TotalMilliseconds:0.0}ms"); + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) + { + lock (events) events.Add(push); + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s {push.NotificationType} seq={push.SequenceId} {push.RawMessage}"); + } + }; + + clock.Restart(); + await scenario.FireAsync(cancellationToken); + + var deadline = clock.Elapsed + TimeSpan.FromSeconds(45); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + log.WriteLine($" +{clock.Elapsed.TotalSeconds,6:0.0}s ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + lock (events) + { + log.WriteLine($" {events.Count} notification(s) over TLS"); + Assert.NotEmpty(events); + + // The TLS half of the endpoint-type derivation, end to end: an encrypted connection asks for an + // FQDN form, so a MOVING should name a *host* rather than an address - which is the whole point, + // since a certificate carrying DNS names cannot validate a bare IP. + foreach (var moving in events.Where(e => e.NotificationType == MaintenanceNotificationType.Moving)) + { + log.WriteLine($" MOVING named: {moving.NewEndPoint?.ToString() ?? "(null)"}"); + if (moving.NewEndPoint is not null) + { + Assert.IsType(moving.NewEndPoint); + } + } + } + + // and the TLS handshake succeeds again on the *replacement* connection, which is the part a + // certificate-name problem would break rather than the first connect + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should re-establish TLS after the endpoint moves"); + } + } +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs new file mode 100644 index 000000000..4c79f1e01 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/TopologyChangeScenarioTests.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// Real topology changes on a real deployment, watched by a real client. +/// +/// +/// Each scenario provisions its own database, because every trigger publishes the dbconfig it requires +/// and all of them want proxy_policy: single - a shape the environment templates do not create. +/// +/// What these assert is deliberately modest: that the notifications arrive, parse, and open the relaxation +/// window. Timings are *recorded* rather than asserted, because the measured spread across clusters is wide +/// enough that a bound tight enough to be interesting would be flaky - DNS has been seen updating 4.4s after +/// MOVING on one cluster and 3s after the socket closed on another. The log is the deliverable for +/// timing; the assertions cover behaviour. +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "topology-change")] +public class TopologyChangeScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper log) + : IClassFixture +{ + /// + /// Records what arrived and when, relative to the moment the scenario was fired. + /// + private sealed class Timeline(Stopwatch clock, ITestOutputHelper log) + { + private readonly List _entries = []; + + public List Events { get; } = []; + + public void Note(string what) + { + var entry = $" +{clock.Elapsed.TotalSeconds,6:0.0}s {what}"; + lock (_entries) + { + _entries.Add(entry); + } + + log.WriteLine(entry); + } + + public void Add(PushMaintenanceEvent evt) + { + lock (_entries) + { + Events.Add(evt); + } + + Note($"{evt.NotificationType} seq={evt.SequenceId} time={evt.Time?.TotalSeconds.ToString() ?? "-"} {evt.RawMessage}"); + } + + public int Count + { + get { lock (_entries) { return Events.Count; } } + } + } + + /// + /// Whether each scenario announces itself, and why - measured 2026-09-01 against RS 8.0.22. + /// + /// + /// The expectations encode the rule the measurements produced, which is narrower than either half of it + /// looks: a connection is told MOVING when its own proxy leaves the endpoint's address set *and* + /// the set gains a member. Both conditions are needed - a pure reduction takes the proxy away without + /// announcing it, and a pure widening adds addresses while leaving the connection's proxy in place, which is + /// also silent. The data-movement pair (MIGRATING/MIGRATED) is separate and fires whenever + /// shards move, whether or not any endpoint changes. + /// + [Theory] + [InlineData("conn_drop", "endpoint_rebind", true)] + [InlineData("dns_resolution_change", "endpoint_rebind", false)] + [InlineData("data_movement_conn_drop", "maintenance_mode", true)] + [InlineData("data_movement_no_conn_drop", "migrate", true)] + public async Task ScenarioProducesNotificationsWeUnderstand(string effect, string trigger, bool expectNotifications) + { + fixture.RequireAvailable(); + var cancellationToken = TestContext.Current.CancellationToken; + + await using var scenario = await ScenarioRun.SetupAsync( + fixture.Injector, "topology-change-standalone", effect, trigger, log.WriteLine, cancellationToken: cancellationToken); + + var database = scenario.Database; + Assert.NotNull(database); + + var clock = Stopwatch.StartNew(); + var timeline = new Timeline(clock, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(database.GetClientConfig(fixture.Environment)); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(conn.GetEndPoints()[0]); + Assert.True(endpoint.MaintenanceNotificationsActive, "the opt-in must be live, or this test proves nothing"); + + conn.ServerMaintenanceEvent += (_, e) => + { + if (e is PushMaintenanceEvent push) timeline.Add(push); + }; + conn.ConnectionFailed += (_, e) => timeline.Note($"connection failed: {e.FailureType} {e.Exception?.Message}"); + conn.ConnectionRestored += (_, e) => timeline.Note("connection restored"); + + clock.Restart(); + timeline.Note($"firing {effect}/{trigger} against {database}"); + await scenario.FireAsync(cancellationToken); + timeline.Note("injector reports the scenario finished"); + + // The injector finishing is not the deployment settling: notifications, the DNS change and the socket + // close all trail it. Keep watching, and keep the client busy so a broken connection actually surfaces + // rather than sitting idle. + var deadline = clock.Elapsed + TimeSpan.FromSeconds(60); + while (clock.Elapsed < deadline) + { + try + { + await conn.GetDatabase().PingAsync(); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + timeline.Note($"ping failed: {ex.GetType().Name}"); + } + + await Task.Delay(1000, cancellationToken); + } + + timeline.Note($"finished with {timeline.Count} notification(s); relaxed={endpoint.IsMaintenanceRelaxed}"); + + if (expectNotifications) + { + Assert.NotEmpty(timeline.Events); + Assert.All(timeline.Events, e => Assert.NotEqual(MaintenanceNotificationType.None, e.NotificationType)); + } + else + { + // dns_resolution_change widens the policy (single -> all-master-shards), so addresses are *added* + // and the connection's own proxy stays where it is - there is nothing to tell this connection to + // move, and the proxy restarting to apply the change closes the socket with no warning at all. + // Asserting the silence deliberately: it is the case with no signal, so if a future build starts + // announcing it, that is a behaviour change we want to be told about rather than to absorb quietly. + Assert.Empty(timeline.Events); + } + + // and the client is usable afterwards, which is the point of the whole feature + Assert.True( + await Poll.UntilAsync(() => TryPing(conn), timeoutMilliseconds: 30_000), + "the client should be serving commands again after the topology change"); + } + + private static bool TryPing(IConnectionMultiplexer conn) + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + } +} diff --git a/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs b/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs new file mode 100644 index 000000000..302c880f1 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/AdvertisedAddressProbeTests.cs @@ -0,0 +1,216 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The DNS probe behind the MOVING handoff. Tested against an injected resolver rather than real DNS, +/// which is the only way to exercise the case that actually happens: the first answer naming the address we +/// were just told to leave. +/// +public class AdvertisedAddressProbeTests(ITestOutputHelper log) +{ + private static readonly IPAddress Retiring = IPAddress.Parse("10.129.228.140"); + private static readonly IPAddress Replacement = IPAddress.Parse("10.252.90.18"); + private static readonly DnsEndPoint Endpoint = new("db.example.cloud.redislabs.com", 13486); + + /// + /// Resolves from a script: one entry per call, the last repeating forever. + /// + private sealed class ScriptedResolver(params IPAddress[][] answers) + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + + public Task ResolveAsync(string host, CancellationToken cancellationToken) + { + var index = Interlocked.Increment(ref _calls) - 1; + return Task.FromResult(answers[Math.Min(index, answers.Length - 1)]); + } + } + + [Fact] + public async Task DnsTrailingTheNotificationIsPolledThrough() + { + // The measured case: DNS named the retiring address for the first 4.4-9.7 seconds after MOVING. A + // client that accepted the first answer would hand off to the node it was told to leave. + var resolver = new ScriptedResolver( + [Retiring], + [Retiring], + [Retiring], + [Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(4, resolver.Calls); + } + + [Fact] + public async Task ReplacementOnTheFirstAnswerIsTakenImmediately() + { + var resolver = new ScriptedResolver([Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Fact] + public async Task WindowExpiringWithoutAMoveGivesUpRatherThanGuessing() + { + // Not a failure to report: the server closes the socket regardless, and the relaxed window covers the + // reconnect. Guessing an address here would be worse than doing nothing. + var resolver = new ScriptedResolver([Retiring]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromMilliseconds(120), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Null(result); + Assert.True(resolver.Calls > 1, $"should have retried within the window, but resolved {resolver.Calls} time(s)"); + } + + [Fact] + public async Task ZeroWindowStillGetsOneAttempt() + { + // "act now" rather than "do nothing": the notifications legitimately carry zero or negative times for + // a connection that arrived mid-window + var resolver = new ScriptedResolver([Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.Zero, pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Fact] + public async Task ResolutionFailureIsRetriedNotFatal() + { + // a DNS blip mid-handoff is when we can least afford to give up + int calls = 0; + Task Resolve(string host, CancellationToken cancellationToken) + { + calls++; + return calls switch + { + 1 => throw new System.Net.Sockets.SocketException(11001), // host not found + 2 => Task.FromResult([Retiring]), + _ => Task.FromResult([Replacement]), + }; + } + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolve, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(3, calls); + } + + [Fact] + public async Task MultipleAddressesTakeTheOneThatIsNotRetiring() + { + // a round-robin record can name both nodes at once mid-move + var resolver = new ScriptedResolver([Retiring, Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + } + + [Fact] + public async Task PlacementNotPolicyDecidesWhetherWeWait() + { + // The A-record count follows actual proxy placement, not the policy name: an all-master-shards + // database whose shards share a node resolves to one address, and then there is no sibling to step to + // and the wait is the only option. Same code path as `single`, which is the point - nothing here reads + // the policy. + var resolver = new ScriptedResolver([Retiring], [Retiring], [Replacement]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(Replacement, 13486), result); + Assert.Equal(3, resolver.Calls); // it waited, because there was nothing else advertised + } + + [Fact] + public async Task SiblingIsTakenWithoutWaitingForTheRecordToMove() + { + // The common case: several A records, so the first resolution already names a live sibling proxy while + // the retiring address is *still* advertised. Stepping sideways immediately is correct - any proxy of + // the same database serves the same data - and it means the poll usually never engages. + var sibling = IPAddress.Parse("10.246.250.155"); + var resolver = new ScriptedResolver([Retiring, sibling]); + + var result = await AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromSeconds(1), + resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(new IPEndPoint(sibling, 13486), result); + Assert.Equal(1, resolver.Calls); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task StillAdvertisedAnswersTheUnannouncedCase(bool present) + { + // The measured gap: a multi-proxy node taken out for maintenance announced only MIGRATING/MIGRATED, + // dropped the victim from DNS at +21.4s, and closed its socket silently at +34.7s. For thirteen + // seconds the condition was visible to anyone who asked, and no notification said so. + var sibling = IPAddress.Parse("10.246.250.155"); + var resolver = new ScriptedResolver(present ? [Retiring, sibling] : [sibling, Replacement]); + + var result = await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: resolver.ResolveAsync, log: log.WriteLine); + + Assert.Equal(present, result); + } + + [Fact] + public async Task ResolutionFailureIsNotAReasonToAbandonAConnection() + { + // null rather than false, deliberately: "cannot tell" must not become "give it up", or a DNS blip + // recycles every healthy connection at once + Task Throws(string host, CancellationToken cancellationToken) + => throw new System.Net.Sockets.SocketException(11001); + + Assert.Null(await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: Throws, log: log.WriteLine)); + + // and the same for a record that momentarily resolves to nothing at all + Assert.Null(await AdvertisedAddressProbe.IsStillAdvertisedAsync( + Endpoint, Retiring, resolve: (_, _) => Task.FromResult([]), log: log.WriteLine)); + } + + [Fact] + public async Task CancellationStopsThePoll() + { + using var cts = new CancellationTokenSource(); + var resolver = new ScriptedResolver([Retiring]); + + var probe = AdvertisedAddressProbe.ProbeAsync( + Endpoint, Retiring, window: TimeSpan.FromMinutes(1), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: resolver.ResolveAsync, log: log.WriteLine, cancellationToken: cts.Token); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => probe); + } +} diff --git a/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs new file mode 100644 index 000000000..13e551c1f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ClusterSlotMigrationUnitTests.cs @@ -0,0 +1,61 @@ +using System.Linq; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The comma-and-range slot form the cluster notifications carry, e.g. 123,456,789-1000. Split out +/// because it is the one part of stage 3 that a wire capture cannot invalidate - it is the same form +/// CLUSTER NODES has always used. +/// +public class ClusterSlotMigrationUnitTests(ITestOutputHelper log) +{ + [Theory] + [InlineData("123", "123-123")] + [InlineData("123,456", "123-123,456-456")] + [InlineData("789-1000", "789-1000")] + [InlineData("123,456,789-1000", "123-123,456-456,789-1000")] + [InlineData("0-16383", "0-16383")] + [InlineData("5-5", "5-5")] + public void ParsesTheSlotForm(string input, string expected) + { + Assert.True(SlotRange.TryParseList(input, out var ranges)); + var actual = string.Join(",", ranges.Select(x => $"{x.From}-{x.To}")); + log.WriteLine($"{input} -> {actual}"); + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("123,,456", "123-123,456-456")] // empty element, skipped + [InlineData("123,", "123-123")] // trailing comma + [InlineData(",123", "123-123")] // leading comma + public void ToleratesEmptyElements(string input, string expected) + { + Assert.True(SlotRange.TryParseList(input, out var ranges)); + Assert.Equal(expected, string.Join(",", ranges.Select(x => $"{x.From}-{x.To}"))); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(",")] + [InlineData("abc")] + [InlineData("12a")] + [InlineData("-5")] // no lower bound + [InlineData("5-")] // no upper bound + [InlineData("1000-789")] // reversed: a server bug, not something to normalize silently + [InlineData("1-2-3")] + [InlineData("123,abc")] // one bad element fails the list: a partial slot set is worse than none + public void RejectsMalformedInput(string? input) + { + Assert.False(SlotRange.TryParseList(input, out var ranges)); + log.WriteLine($"'{input}' rejected, {ranges.Count} range(s)"); + } + + [Fact] + public void OutOfRangeSlotIsRejected() + { + // 16384 does not fit the short, and a checked cast would throw rather than wrap + Assert.False(SlotRange.TryParseList("99999", out _)); + } +} diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 6c62e3bb2..e0cbb8c0a 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -62,6 +62,11 @@ orderby name Assert.Equal( new[] { + "_maintenanceMovingEndpointType", + "_maintenanceNotifications", + "_maintenancePostEventRelaxedDuration", + "_maintenanceRelaxedTimeout", + "_maintenanceRelaxedWindowMax", "_protocol", "asyncTimeout", "backlogPolicy", @@ -106,6 +111,7 @@ orderby name "sslProtocols", "syncTimeout", "tieBreaker", + "topologyRefreshSeconds", "Tunnel", "user", }, @@ -891,6 +897,68 @@ public void CheckHighIntegrity(bool? assigned, bool expected, string cs) Assert.Equal(expected, parsed.HighIntegrity); } + [Theory] + [InlineData("maintRelaxedTimeout=20", 20, 60, 40)] + [InlineData("maintRelaxedWindowMax=45", 10, 45, 20)] + [InlineData("maintPostEventRelaxed=0", 10, 30, 0)] + public void MaintenanceDurationsRoundTrip(string cs, int relaxedSeconds, int capSeconds, int tailSeconds) + { + // these are in *seconds*, unlike every other timeout here, because that is the unit the cross-client + // contract names for maintRelaxedTimeout - so a documented value can be pasted between clients + var options = Parse("dummy," + cs); + Assert.Equal(TimeSpan.FromSeconds(relaxedSeconds), options.MaintenanceRelaxedTimeout); + Assert.Equal(TimeSpan.FromSeconds(capSeconds), options.MaintenanceRelaxedWindowMax); + Assert.Equal(TimeSpan.FromSeconds(tailSeconds), options.MaintenancePostEventRelaxedDuration); + + // only the explicitly-set key is serialized; the others stay defaulted + Assert.Equal("dummy," + cs, RemoveTestDefaults(options.ToString())); + + var clone = options.Clone(); + Assert.Equal(options.MaintenanceRelaxedTimeout, clone.MaintenanceRelaxedTimeout); + Assert.Equal("dummy," + cs, RemoveTestDefaults(clone.ToString())); + } + + [Fact] + public void MaintenanceDurationsDefaultRelativeToTheRelaxedTimeout() + { + // the cap and tail are multiples so that raising the relaxed timeout cannot produce a cap below it + var options = Parse("dummy,maintRelaxedTimeout=60"); + Assert.Equal(TimeSpan.FromSeconds(180), options.MaintenanceRelaxedWindowMax); + Assert.Equal(TimeSpan.FromSeconds(120), options.MaintenancePostEventRelaxedDuration); + } + + [Fact] + public void MaintenanceDurationInMillisecondsIsDiagnosed() + { + // the silent-misconfiguration case: somebody assumes milliseconds like every other timeout here and + // writes 30000, which as seconds would be an eight-hour relaxed timeout. The message names the unit + var ex = Assert.Throws(() => Parse("dummy,maintRelaxedTimeout=30000")); + Output.WriteLine(ex.Message); + Assert.Contains("seconds", ex.Message); + Assert.Contains("not in milliseconds", ex.Message); + } + + [Theory] + [InlineData(null, MaintenanceNotificationMode.Disabled, "dummy")] + [InlineData(MaintenanceNotificationMode.Disabled, MaintenanceNotificationMode.Disabled, "dummy,maintNotifications=Disabled")] + [InlineData(MaintenanceNotificationMode.Enabled, MaintenanceNotificationMode.Enabled, "dummy,maintNotifications=Enabled")] + [InlineData(MaintenanceNotificationMode.Auto, MaintenanceNotificationMode.Auto, "dummy,maintNotifications=Auto")] + public void CheckMaintenanceNotifications(MaintenanceNotificationMode? assigned, MaintenanceNotificationMode expected, string cs) + { + var options = Parse("dummy"); + if (assigned.HasValue) options.MaintenanceNotifications = assigned.Value; + + Assert.Equal(expected, options.MaintenanceNotifications); + Assert.Equal(cs, RemoveTestDefaults(options.ToString())); + + var clone = options.Clone(); + Assert.Equal(expected, clone.MaintenanceNotifications); + Assert.Equal(cs, RemoveTestDefaults(clone.ToString())); + + var parsed = Parse(cs); + Assert.Equal(expected, parsed.MaintenanceNotifications); + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -902,6 +970,12 @@ public void DefaultsProviderProtocolNotSerialized(bool clone) if (clone) options = options.Clone(); Assert.Equal(RedisProtocol.Resp3, options.Protocol); Assert.Same(provider, options.Defaults); - Assert.Equal("", options.ToString()); + + // the *values* a provider supplies are still never serialized - that is what this test is for... + Assert.DoesNotContain("protocol=", options.ToString()); + + // ...but the explicit choice of provider now is, by name, so the string describes the configuration it + // was given rather than quietly dropping part of it + Assert.Equal("defaults=amr", options.ToString()); } } diff --git a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs new file mode 100644 index 000000000..102e83c17 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis.Configuration; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Re-reading the topology when an endpoint will not accept a connection at all. +/// +/// +/// The gap: every other path that re-reads the topology needs somebody *else* to notice first - a maintenance +/// notification, a MOVED from a reachable node, a peer's configuration broadcast. A client with quiet +/// healthy connections and one endpoint that only ever refuses has nobody to tell it, and +/// reconfigureNextFailure is set only once a connection has been *established*, so a node that never +/// established could be retried indefinitely. +/// +/// Not hypothetical: a customer's client dialled three endpoints that no longer existed for 37 hours across a +/// Redis Cloud node replacement, recovering only when something unrelated finally provoked a re-read. +/// +/// +[Collection(NonParallelCollection.Name)] +public class ConnectFailureRefreshTests(ITestOutputHelper log) +{ + /// Counts inbound CLUSTER commands, so a test can see a topology read happen. + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0 && string.Equals(request.GetString(0), "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + + return base.Execute(client, in request); + } + } + + private (CountingServer Server, ConfigurationOptions Config, EndPoint Doomed, CapturingLogger Logger) Arrange(int configCheckSeconds) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + var doomed = BlackHoleTunnel.GetRefusingEndPoint(); + + // a real member of the topology - it holds a slot, and CLUSTER SLOTS advertises it - that simply + // cannot be reached; the client learns about it from the healthy node and then dials it forever + server.AddEmptyNode(doomed); + server.Migrate((RedisKey)"leaving", doomed); + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.AbortOnConnectFail = false; + config.ConfigCheckSeconds = configCheckSeconds; // the refresh rate limit reuses this + config.ConnectTimeout = 2000; + config.ReconnectRetryPolicy = new LinearRetry(500); // else the default backoff makes this a minutes-long test + var tunnel = new BlackHoleTunnel(server.Tunnel); + tunnel.BlackHole(doomed); // refused from the outset: this endpoint never accepts a connection at all + config.Tunnel = tunnel; + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + return (server, config, doomed, logger); + } + + [Fact] + public async Task AnEndpointThatOnlyEverRefusesProvokesATopologyRead() + { + var (server, config, doomed, logger) = Arrange(configCheckSeconds: 5); + using (server) + { + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + var before = server.ClusterCommands; + log.WriteLine($"cluster commands before: {before}"); + + var refreshed = await Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 30_000); + + // dumped before the assertion, so a failure arrives with the evidence rather than just a verdict + log.WriteLine($"cluster commands after: {server.ClusterCommands}"); + log.WriteLine(logger.All); + Assert.True(refreshed, "repeated connect failures should have provoked a topology read"); + Assert.NotEmpty(logger.Matching("consecutive connect failures")); + } + } + + [Fact] + public async Task TheReadIsRateLimitedRatherThanOncePerFailure() + { + // The restraint is the part that makes this safe to do at all. The gate it replaces exists to prevent a + // stampede - a dead endpoint, times a retry loop, times every client in a fleet, each issuing a + // topology read - so trading a stuck client for a thundering herd would be no improvement. + const int WindowSeconds = 12, ConfigCheckSeconds = 5; + var (server, config, doomed, logger) = Arrange(ConfigCheckSeconds); + using (server) + { + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000)); + + await Task.Delay(TimeSpan.FromSeconds(WindowSeconds), TestContext.Current.CancellationToken); + + var attempts = logger.Matching("Resurrecting").Count; + var refreshes = logger.Matching("consecutive connect failures").Count; + log.WriteLine($"{attempts} connect attempts, {refreshes} topology reads in {WindowSeconds}s"); + + // the ratio is the assertion: many failures, few reads. The bound is generous because the + // heartbeat that drives both is only ~1s accurate, but it is nowhere near one-read-per-failure. + Assert.True(attempts >= 5, $"expected the endpoint to be retried repeatedly, but saw {attempts} attempts"); + var permitted = (WindowSeconds / ConfigCheckSeconds) + 2; + Assert.True(refreshes <= permitted, $"expected at most {permitted} rate-limited reads, but saw {refreshes}"); + } + } + + private sealed class CapturingLogger : ILoggerFactory, ILogger + { + private readonly List _messages = []; + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_messages) _messages.Add(formatter(state, exception)); + } + + public List Matching(string fragment) + { + lock (_messages) return _messages.FindAll(x => x.Contains(fragment, StringComparison.Ordinal)); + } + + public string All + { + get { lock (_messages) return string.Join("\n", _messages); } + } + + public void Dispose() { } + } +} diff --git a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs index fae837bd9..7da6ab839 100644 --- a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs +++ b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs @@ -86,6 +86,135 @@ public void IsMatchOnAzureManagedRedisDomain(string hostName) Assert.IsType(provider); } + [Theory] + [InlineData("amr", typeof(AzureManagedRedisOptionsProvider))] + [InlineData("AMR", typeof(AzureManagedRedisOptionsProvider))] // names are case-insensitive, like every other key + [InlineData("azure", typeof(AzureOptionsProvider))] + [InlineData("rediscloud", typeof(RedisCloudOptionsProvider))] + [InlineData("enterprise", typeof(RedisEnterpriseOptionsProvider))] + public void DefaultsProviderCanBeNamedInAConfigurationString(string name, Type expected) + { + // the on-premise case is why this exists: an Enterprise cluster has whatever DNS its operator gave it, + // so IsMatch can never recognize it, and until now the only way to select a provider was to write code + var options = ConfigurationOptions.Parse($"localhost,defaults={name}"); + Assert.IsType(expected, options.Defaults); + + // and it round-trips, because it was chosen rather than inferred + var text = options.ToString(); + Output.WriteLine(text); + Assert.Contains($"defaults={name.ToLowerInvariant()}", text); + Assert.IsType(expected, ConfigurationOptions.Parse(text).Defaults); + } + + [Fact] + public void ProviderToStringPrefersTheName() + { + // for logs: "amr" rather than a namespace-qualified type name + Assert.Equal("amr", new AzureManagedRedisOptionsProvider().ToString()); + Assert.Equal("enterprise", new RedisEnterpriseOptionsProvider().ToString()); + + // ...and an unnameable one still says something useful, which is exactly why serialization tests + // Name rather than ToString: this is never null + var custom = new TestOptionsProvider(".custom"); + Assert.Null(custom.Name); + Assert.Contains(nameof(TestOptionsProvider), custom.ToString()); + } + + [Fact] + public void UnknownDefaultsProviderNameIsRejectedWithTheAlternatives() + { + var ex = Assert.Throws(() => ConfigurationOptions.Parse("localhost,defaults=sideways")); + Output.WriteLine(ex.Message); + Assert.Contains("enterprise", ex.Message); // the message lists what it would have accepted + Assert.Contains("rediscloud", ex.Message); + } + + [Fact] + public void AnInferredDefaultsProviderIsNotSerialized() + { + // the trap this guards: the Defaults getter memoizes an inferred provider into the same field an + // explicit set writes, so without tracking *how* it got there, merely reading the property would make + // an endpoint-derived guess look like a decision - and re-parsing the string would then pin it + var options = ConfigurationOptions.Parse("contoso.cloud.redislabs.com"); + Assert.IsType(options.Defaults); // inferred, and memoized by that read + + Assert.DoesNotContain("defaults=", options.ToString()); + Assert.DoesNotContain("defaults=", options.Clone().ToString()); + } + + [Fact] + public void AnExplicitDefaultsProviderSurvivesCloning() + { + var options = ConfigurationOptions.Parse("localhost,defaults=enterprise"); + var clone = options.Clone(); + + Assert.IsType(clone.Defaults); + Assert.Contains("defaults=enterprise", clone.ToString()); + } + + [Fact] + public void AnUnnameableProviderIsNotSerializedEvenWhenExplicit() + { + // a custom provider is still perfectly usable in code; it just cannot be expressed as a string, the + // same way an inbuilt tunnel can be and a custom one cannot + var options = ConfigurationOptions.Parse("localhost"); + options.Defaults = new TestOptionsProvider(".unnameable"); + + Assert.Null(options.Defaults.Name); + Assert.DoesNotContain("defaults=", options.ToString()); + } + + [Theory] + [InlineData("redis-12345.c1.eu-west-1-2.ec2.cloud.redislabs.com")] + [InlineData("contoso.CLOUD.REDISLABS.COM")] // case-insensitive, as the sibling providers are + [InlineData("redis-12345.c1.us-east-1-2.ec2.cloud.redis.io")] // newer routing scheme + [InlineData("contoso.redislabs.com")] // older subscriptions, addressed directly + public void IsMatchOnRedisCloudDomain(string hostName) + { + var epc = new EndPointCollection(new List() { new DnsEndPoint(hostName, 0) }); + var provider = DefaultOptionsProvider.GetProvider(epc); + Assert.IsType(provider); + } + + [Fact] + public void RedisCloudDoesNotInheritTheAzureManagedAssumptions() + { + // the two deployments look similar and are not: AMR is TLS-only with a 7.4 floor, Redis Cloud is + // neither, so copying those two would break plaintext databases and over-claim the server version + var epc = new EndPointCollection(new List() { new DnsEndPoint("contoso.cloud.redislabs.com", 0) }); + var cloud = DefaultOptionsProvider.GetProvider(epc); + var amr = new AzureManagedRedisOptionsProvider(); + + Assert.True(amr.GetDefaultSsl(epc)); + Assert.False(cloud.GetDefaultSsl(epc)); + Assert.Equal(RedisFeatures.v7_4_0, amr.DefaultVersion); + Assert.Equal(DefaultOptionsProvider.BaseDefaultVersion, cloud.DefaultVersion); + + // ...but it does share the parts that are about being a proxied, hosted deployment + Assert.Equal(RedisProtocol.Resp3, cloud.Protocol); + Assert.Equal("", cloud.ConfigurationChannel); + Assert.False(cloud.AbortOnConnectFail); + } + + [Theory] + [InlineData("contoso.redis.azure.net", MaintenanceNotificationMode.Auto)] // AMR asks, pre-emptively + [InlineData("contoso.cloud.redislabs.com", MaintenanceNotificationMode.Auto)] // and Redis Cloud, which emits them + [InlineData("contoso.redis.cache.windows.net", MaintenanceNotificationMode.Disabled)] // classic Azure does not + [InlineData("contoso.example.com", MaintenanceNotificationMode.Disabled)] // and neither does anything else + public void MaintenanceNotificationDefaultPerProvider(string hostName, MaintenanceNotificationMode expected) + { + // Auto rather than Enabled is what makes the pre-emptive default safe: AMR does not emit these yet, + // so until the server side ships the opt-in is refused and the feature simply stays off + var epc = new EndPointCollection(new List() { new DnsEndPoint(hostName, 0) }); + var provider = DefaultOptionsProvider.GetProvider(epc); + Output.WriteLine($"{hostName} -> {provider.GetType().Name}"); + Assert.Equal(expected, provider.MaintenanceNotifications); + + // ...and it arrives through the options, not just off the provider + var options = new ConfigurationOptions { EndPoints = { new DnsEndPoint(hostName, 0) } }; + Assert.Equal(expected, options.MaintenanceNotifications); + } + [Theory] [InlineData(RedisProtocol.Resp2)] [InlineData(RedisProtocol.Resp3)] diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index c8a30030e..0c2d4892f 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using StackExchange.Redis.Availability; using Xunit; @@ -6,6 +6,34 @@ namespace StackExchange.Redis.Tests; public class HealthCheckPolicyUnitTests { + /// + /// Probe traffic must not be counted as work a caller is waiting on, because idleness is what decides + /// whether an endpoint that has left the deployment can be given up - a probe that looks like caller work + /// makes a server appear busy precisely because we are watching it. + /// + /// + /// The other half of this test is what it asserts is *false*: the probe must not become an internal call, + /// which would also change how it is queued - internal calls bypass the backlog. A health check that + /// bypasses the backlog cannot observe a bridge whose queue is not draining, which is exactly the signal + /// geo-redundant failover depends on. + /// + [Fact] + public void ProbeTrafficIsNotCallerFacingButIsRoutedNormally() + { + var flags = new HealthCheckContext(null!, TimeSpan.FromSeconds(1)).ProbeFlags; + Assert.NotEqual(CommandFlags.None, flags); + + var message = Message.Create(-1, flags, RedisCommand.PING); + Assert.True(message.IsProbe); + Assert.False(message.IsCallerFacing); + Assert.False(message.IsInternalCall); + + // and an ordinary command is unaffected in both directions + var ordinary = Message.Create(-1, CommandFlags.None, RedisCommand.PING); + Assert.False(ordinary.IsProbe); + Assert.True(ordinary.IsCallerFacing); + } + [Theory] [InlineData(0, 0, 5, HealthCheckResult.Inconclusive)] // No results yet [InlineData(1, 0, 0, HealthCheckResult.Healthy)] // One success, no more probes diff --git a/tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs b/tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs new file mode 100644 index 000000000..fd065c802 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/Helpers/BlackHoleTunnel.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Configuration; + +namespace StackExchange.Redis.Tests; + +/// +/// Wraps another and sends chosen endpoints to a real socket instead, so they are +/// refused. +/// +/// +/// The fixture for "a node that has gone away", which is harder to model than it looks. Removing a node from +/// the in-process fake is not enough: InProcTunnel only intercepts endpoints TryGetNode still +/// resolves, but an already-established in-process pipe survives the removal, so the client never reconnects +/// and therefore never fails. Falling through to a real socket against a loopback port that was bound and +/// released gives connection-refused on every attempt, which is what a departed node actually does. +/// +/// Endpoints can be black-holed after connecting, which is what lets a test establish a connection, get the +/// client into the state it wants, and only then take the node away. +/// +/// +internal sealed class BlackHoleTunnel(Tunnel inner) : Tunnel +{ + private readonly HashSet _blackHoled = []; + + /// A loopback port that has been bound and released, so connecting to it is refused rather than dropped. + /// + /// Refused rather than timing out is the point: a dropped SYN would exercise connect *timeouts*, which is + /// a different failure mode with very different timing, and would make any test built on it slow. + /// + public static IPEndPoint GetRefusingEndPoint() + { + using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + probe.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return (IPEndPoint)probe.LocalEndPoint!; + } + + /// Stops intercepting this endpoint, so connecting to it is refused from now on. + public void BlackHole(EndPoint endpoint) + { + lock (_blackHoled) _blackHoled.Add(endpoint); + } + + private bool IsBlackHoled(EndPoint endpoint) + { + lock (_blackHoled) return _blackHoled.Contains(endpoint); + } + + public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) + => IsBlackHoled(endpoint) + ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) + : inner.GetSocketConnectEndpointAsync(endpoint, cancellationToken); + + public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) + => IsBlackHoled(endpoint) + ? base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) + : inner.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); +} diff --git a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs index 09947fa25..7d793f18a 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs @@ -26,6 +26,32 @@ public static class TestConfig public static int MinTimeoutMilliseconds { get; } = int.TryParse(Environment.GetEnvironmentVariable("REDIS_TESTS_MIN_TIMEOUT_MS"), out var ms) && ms > 0 ? ms : 0; + /// + /// A suite-wide maintenance-notification mode, from REDIS_TESTS_MAINT_NOTIFICATIONS + /// (disabled/enabled/auto); null (the default) leaves the library's own + /// default alone. + /// + /// + /// The point is to be able to run the *whole* suite with the opt-in on, rather than to test the feature: + /// most servers we test against never send a notification, and that is the expected outcome - the opt-in + /// must be invisible to everything else. Applied only where a test has not asked for a specific mode. + /// + public static MaintenanceNotificationMode? MaintenanceNotifications { get; } = + Enum.TryParse(Environment.GetEnvironmentVariable("REDIS_TESTS_MAINT_NOTIFICATIONS"), ignoreCase: true, out var mode) + ? mode : null; + + /// + /// Applies , if set. Call before any test-specific override. + /// + public static ConfigurationOptions ApplyMaintenanceDefault(ConfigurationOptions config) + { + if (MaintenanceNotifications is { } mode) + { + config.MaintenanceNotifications = mode; + } + return config; + } + #if NET private static int _db = 17; #else diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 4d27a206b..2276500bd 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -53,6 +53,14 @@ public InProcessTestServer(ITestOutputHelper? log = null, EndPoint? endpoint = n Tunnel = new InProcTunnel(this); } + /// + /// The highest protocol this server will agree to; lowering it lets a test have a HELLO 3 accepted + /// and answered as RESP2, which is the shape a client has to survive without being told about it. + /// + public RedisProtocol MaxProtocolVersion { get; set; } = RedisProtocol.Resp3; + + protected override RedisProtocol MaxProtocol => MaxProtocolVersion; + public Task ConnectAsync(bool withPubSub = true, bool defaultOnly = false, WriteMode writeMode = WriteMode.Default, TextWriter? log = null) => ConnectionMultiplexer.ConnectAsync(GetClientConfig(withPubSub, defaultOnly, writeMode), log); @@ -116,6 +124,7 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default Protocol = TestContext.Current.GetProtocol(), // WriteMode = (BufferedStreamWriter.WriteMode)writeMode, }; + TestConfig.ApplyMaintenanceDefault(config); if (!string.IsNullOrEmpty(Password)) config.Password = Password; config.Ssl = UseSsl; // explicitly, ignore provider defaults if (UseSsl) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs new file mode 100644 index 000000000..d240626cb --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceEndpointTypeResolverTests.cs @@ -0,0 +1,60 @@ +using System.Net; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Deriving which moving-endpoint-type to ask for. +/// +/// +/// Worth testing exhaustively because it decides what a server sends us during a handoff, and getting the +/// scope wrong means being handed an address we cannot reach while getting the form wrong means being handed one +/// we cannot validate. +/// +public class MaintenanceEndpointTypeResolverTests +{ + [Theory] + // private, so the internal forms; TLS decides ip versus fqdn + [InlineData("10.0.0.1", false, MaintenanceEndpointType.InternalIp)] + [InlineData("10.0.0.1", true, MaintenanceEndpointType.InternalFqdn)] + // public, so the external forms + [InlineData("34.253.226.6", false, MaintenanceEndpointType.ExternalIp)] + [InlineData("34.253.226.6", true, MaintenanceEndpointType.ExternalFqdn)] + public void ScopeComesFromTheAddressAndFormFromTls(string address, bool encrypted, MaintenanceEndpointType expected) + => Assert.Equal(expected, MaintenanceEndpointTypeResolver.Derive(IPAddress.Parse(address), encrypted)); + + [Fact] + public void NoAddressMeansAskForNothing() + { + // A tunnel, a custom transport or a Unix domain socket gives us nothing to classify. "none" is the + // honest answer: we ask for no address and reconnect the way we originally connected, rather than + // guessing at a scope we cannot determine. + Assert.Equal(MaintenanceEndpointType.None, MaintenanceEndpointTypeResolver.Derive(null, isEncrypted: false)); + Assert.Equal(MaintenanceEndpointType.None, MaintenanceEndpointTypeResolver.Derive(null, isEncrypted: true)); + } + + [Theory] + [InlineData("10.0.0.1", true)] // RFC1918 10/8 + [InlineData("10.255.255.255", true)] + [InlineData("172.16.0.1", true)] // RFC1918 172.16/12 - lower edge + [InlineData("172.31.255.254", true)] // upper edge + [InlineData("172.15.0.1", false)] // just outside + [InlineData("172.32.0.1", false)] // just outside + [InlineData("192.168.1.1", true)] // RFC1918 192.168/16 + [InlineData("192.169.1.1", false)] // adjacent, and public + [InlineData("169.254.1.1", true)] // link-local + [InlineData("127.0.0.1", true)] // loopback + [InlineData("::1", true)] // IPv6 loopback + [InlineData("fe80::1", true)] // IPv6 link-local + [InlineData("fc00::1", true)] // IPv6 unique-local, lower edge of fc00::/7 + [InlineData("fdff::1", true)] // upper edge + [InlineData("fe00::1", false)] // outside fc00::/7 + [InlineData("::ffff:10.0.0.1", true)] // IPv4-mapped private: must be unwrapped, or it reads as public + [InlineData("::ffff:34.253.226.6", false)] // IPv4-mapped public + [InlineData("2001:4860:4860::8888", false)] // public IPv6 + [InlineData("34.253.226.6", false)] // public IPv4 + [InlineData("100.64.0.1", false)] // CGNAT: deliberately *not* private - see the remarks + public void ReservedRangesAreClassified(string address, bool expected) + => Assert.Equal(expected, MaintenanceEndpointTypeResolver.IsPrivateOrReserved(IPAddress.Parse(address))); +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs new file mode 100644 index 000000000..2326e3288 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceHandoffTests.cs @@ -0,0 +1,133 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Deciding what a MOVING means: where to go, or that there is nowhere to go. +/// +/// +/// Tested as a pure decision, with the resolver supplied, because the whole thing turns on DNS *changing* - +/// which no in-process fake can arrange. +/// +public class MaintenanceHandoffTests(ITestOutputHelper log) +{ + private static readonly IPAddress Retiring = IPAddress.Parse("10.129.228.140"); + private static readonly IPAddress Replacement = IPAddress.Parse("10.252.90.18"); + private static readonly DnsEndPoint Hostname = new("db.example.cloud.redislabs.com", 13486); + + private static Func> Resolves(params IPAddress[] addresses) + => (_, _) => Task.FromResult(addresses); + + [Fact] + public async Task HostnameThatHasMovedIsRecycled() + { + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.Recycle, decision.Action); + Assert.Equal(new IPEndPoint(Replacement, 13486), decision.Target); + } + + [Fact] + public async Task HostnameThatNeverMovesDoesNothing() + { + // Not a failure: the server closes the socket regardless, the reconnect re-resolves, and the relaxed + // window covers the gap. Measured on a real cluster - DNS updated three seconds *after* the close - so + // this is a normal outcome rather than a defensive branch. + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromMilliseconds(100), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: Resolves(Retiring), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.None, decision.Action); + Assert.Null(decision.Target); + } + + [Fact] + public async Task AddressEndpointWithNoSuccessorReconnectsOnTheClock() + { + // An address cannot be re-resolved and nothing was named, so a change is undetectable from here - which + // is exactly the case the contract's half-window rule was written for. The address may be a stable + // front for a backend that has already moved, and waiting passively means being closed mid-command + // instead of choosing the moment. + var decision = await MaintenanceHandoff.DecideAsync( + new IPEndPoint(Retiring, 13486), successor: null, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.RecycleAtHalfWindow, decision.Action); + } + + [Fact] + public async Task NamedSuccessorIsUsedDirectly() + { + // A named successor skips DNS entirely, which is the point of the field: DNS trails a MOVING by 4.4s to + // 18.7s while the socket closes at 15.7s to 19.1s, so waiting for it is sometimes waiting too long. + // Note the resolver here still reports the *old* address, and it is never consulted. + var successor = new IPEndPoint(Replacement, 13486); + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor, currentAddress: Retiring, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Retiring), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.MoveTo, decision.Action); + Assert.Equal(successor, decision.Target); + } + + [Fact] + public async Task UnknownCurrentAddressReconnectsOnTheClockRatherThanPolling() + { + // With no idea where we are, "has it moved?" is unanswerable, so there is nothing to poll for. A + // tunnel or a Unix domain socket lands here, and for a tunnel the target genuinely may have moved + // underneath us - so the half-window reconnect is the only tool, and better than waiting to be closed. + var decision = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: null, + window: TimeSpan.FromSeconds(5), pollInterval: TimeSpan.FromMilliseconds(10), + resolve: Resolves(Replacement), log: log.WriteLine); + + log.WriteLine(decision.ToString()); + Assert.Equal(HandoffAction.RecycleAtHalfWindow, decision.Action); + } + + [Fact] + public async Task PollingBeatsTheClockWhenWeCanSeeTheAddress() + { + // The deliberate divergence from the contract's half-window rule, and the reason for it. Where DNS can + // be polled we wait for it to actually move rather than reconnecting on a timer: measured across three + // runs, DNS had moved by half of a 15s window only once (+4.4s), and lagged well past it otherwise + // (+9.7s, +18.7s) - so reconnecting on the clock would usually land back on the node being retired. + // Doing nothing when it never moves is also deliberate: the server closes the socket, the reconnect + // re-resolves, and the relaxed window covers the gap. + var stillOld = await MaintenanceHandoff.DecideAsync( + Hostname, successor: null, currentAddress: Retiring, + window: TimeSpan.FromMilliseconds(120), pollInterval: TimeSpan.FromMilliseconds(20), + resolve: Resolves(Retiring), log: log.WriteLine); + + Assert.Equal(HandoffAction.None, stillOld.Action); + } + + [Theory] + [InlineData(15, 0, 1000)] // a generous window: capped at a second, not a tenth of fifteen + [InlineData(2, 0, 200)] // a short one: a tenth, so a 2s window never spends more than 200ms + [InlineData(0, 0, 0)] // "act now" + public void JitterScalesWithTheWindowAndIsCapped(int windowSeconds, int minMilliseconds, int maxMilliseconds) + { + var random = new Random(12345); + for (int i = 0; i < 200; i++) + { + var jitter = MaintenanceHandoff.GetJitter(TimeSpan.FromSeconds(windowSeconds), random); + Assert.InRange(jitter.TotalMilliseconds, minMilliseconds, maxMilliseconds); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs new file mode 100644 index 000000000..a399312ce --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -0,0 +1,1003 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Receiving the notifications: can the client see them at all, and does it make sense of the payload. Nothing +/// here asserts a *reaction* - there isn't one yet, deliberately - only that a push frame the server sends +/// arrives as an event with the right contents, and that a malformed one is dropped without collateral damage. +/// +public class MaintenanceNotificationTests(ITestOutputHelper log) +{ + private const int DefaultTimeoutMilliseconds = 5000; + + /// + /// The notifications are RESP3 push frames, so every test here forces RESP3 rather than running per + /// protocol: under RESP2 there is nothing to receive, which is covered by the opt-in tests instead. + /// + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( + ITestOutputHelper log, + ILoggerFactory? loggerFactory = null) + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; // must be live, or the test is vacuous + config.LoggerFactory = loggerFactory; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn, new EventCollector(conn)); + } + + /// Captures the library's own log lines, so a test can assert on one it emits. + private sealed class CapturingLoggerFactory : ILoggerFactory, ILogger + { + private readonly List _lines = []; + + public IReadOnlyList Lines + { + get { lock (_lines) return [.. _lines]; } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_lines) _lines.Add(formatter(state, exception)); + } + + public void Dispose() { } + } + + private sealed class EventCollector + { + private readonly ConcurrentQueue _events = new(); + + public EventCollector(IConnectionMultiplexer conn) + => conn.ServerMaintenanceEvent += (_, e) => _events.Enqueue(e); + + public int Count => _events.Count; + + public IReadOnlyList All => _events.ToArray(); + + public async Task NextAsync(int timeoutMilliseconds = DefaultTimeoutMilliseconds) + { + for (int i = 0; i < timeoutMilliseconds / 25; i++) + { + if (_events.TryDequeue(out var next)) return Assert.IsType(next); + await Task.Delay(25); + } + + throw new TimeoutException("No maintenance event was received"); + } + + /// + /// Deliberately proves a negative, so it has to wait out a window in which one could have arrived. + /// + public async Task AssertNoneAsync(int milliseconds = 250) + { + await Task.Delay(milliseconds); + Assert.Empty(All); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating, MaintenanceNotificationType.Migrating)] + [InlineData(MaintenanceNotificationKind.Migrated, MaintenanceNotificationType.Migrated)] + [InlineData(MaintenanceNotificationKind.FailingOver, MaintenanceNotificationType.FailingOver)] + [InlineData(MaintenanceNotificationKind.FailedOver, MaintenanceNotificationType.FailedOver)] + public async Task ShardNotificationIsReceived(MaintenanceNotificationKind sent, MaintenanceNotificationType expected) + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var seq = server.NextMaintenanceSequenceId; + Assert.Equal(1, server.SendShardNotification(null, sent, timeSeconds: 12, shardIds: "[\"shard:1\"]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(expected, evt.NotificationType); + Assert.Equal(seq, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(12), evt.Time); + Assert.Equal("[\"shard:1\"]", evt.Payload); // opaque, carried through verbatim + Assert.Equal(server.DefaultEndPoint, evt.EndPoint); + Assert.Null(evt.NewEndPoint); + } + } + + [Fact] + public async Task MovingCarriesItsSuccessor() + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var target = new IPEndPoint(IPAddress.Loopback, 7999); + Assert.Equal(1, server.SendMoving(null, timeSeconds: 15, newEndpoint: target)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Moving, evt.NotificationType); + Assert.Equal(TimeSpan.FromSeconds(15), evt.Time); + Assert.Equal(target, evt.NewEndPoint); + + // and the deadline is projected forward, which is what a consumer actually wants + Assert.NotNull(evt.StartTimeUtc); + Assert.Equal(evt.ReceivedTimeUtc.AddSeconds(15), evt.StartTimeUtc); + } + } + + [Fact] + public async Task MovingWithNoAddressIsStillReported() + { + // the documented no-address form: a client must handle it whether or not it asked for "none", and the + // intended handling is to reconnect to what it already has - so the event must arrive, not be dropped + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendMoving(null, timeSeconds: 15, newEndpoint: null)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Moving, evt.NotificationType); + Assert.Null(evt.NewEndPoint); + Assert.Null(evt.Payload); + } + } + + [Theory] + [InlineData("?:7002")] // unknown node + [InlineData(":7002")] // node does not know its own address + [InlineData("host-1.example.com:0")] // no port to dial + public async Task MovingWithAPlaceholderYieldsNoEndpoint(string placeholder) + { + // none of these can be dialled, and none of them may be read as "the server that told us" - so the + // event arrives with the raw text and no endpoint, rather than with a plausible-looking wrong one + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MOVING", "1", "15", placeholder)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Null(evt.NewEndPoint); + Assert.Equal(placeholder, evt.Payload); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.SlotMigrating, MaintenanceNotificationType.SlotMigrating)] + [InlineData(MaintenanceNotificationKind.SlotMigrated, MaintenanceNotificationType.SlotMigrated)] + public async Task SlotNotificationIsReceived(MaintenanceNotificationKind sent, MaintenanceNotificationType expected) + { + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotNotification(null, sent, slots: "123,456,789-1000")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(expected, evt.NotificationType); + Assert.Equal("123,456,789-1000", evt.Payload); + Assert.Null(evt.Time); // these carry no duration + } + } + + [Fact] + public async Task AllDigitSlotListIsNotMistakenForADuration() + { + // why the type decides whether a time is expected, rather than the content: a single-slot list is + // indistinguishable from a duration by inspection + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, slots: "123")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal("123", evt.Payload); + Assert.Null(evt.Time); + } + } + + [Fact] + public async Task IntegersMayArriveAsStrings() + { + // "accept $ or : for integers" - the contract says so explicitly, and SendRawPush writes bulk strings + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MIGRATING", "42", "7", "[\"shard:2\"]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(42, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(7), evt.Time); + } + } + + [Fact] + public async Task NegativeTimeIsPreserved() + { + // a connection that joins mid-window is told what is left of it, which can be negative: that means + // "act now", so it must not be clamped or rejected + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: -3)); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(TimeSpan.FromSeconds(-3), evt.Time); + Assert.Null(evt.StartTimeUtc); // nothing sensible to project + } + } + + [Fact] + public async Task LowercaseTypeNamesAreRecognized() + { + // no specification is careful about casing, so neither are we + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "failing_over", "5", "9", "[]")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.FailingOver, evt.NotificationType); + } + } + + [Fact] + public async Task TrailingElementsAreTolerated() + { + // forwards compatibility is a stated requirement: a future field must not cost us the notification + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendRawPush(null, "MIGRATING", "8", "20", "[\"shard:3\"]", "something-new", "and-another")); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Migrating, evt.NotificationType); + Assert.Equal(8, evt.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(20), evt.Time); + Assert.Equal("[\"shard:3\"]", evt.Payload); + } + } + + [Fact] + public async Task NestedSlotMigrationsAreRead() + { + // the shape the shipped clients read: [type, seq, [[source, target, slots], ...]]. We were dropping + // this until the prior-art cross-check, because the parser rejected non-scalar elements + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.Equal(1, server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7000", "127.0.0.1:7001", "0-99"), + ("127.0.0.1:7002", "127.0.0.1:7003", "1000,2000-2500"), + ])); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrated, evt.NotificationType); + Assert.Equal(2, evt.SlotMigrations.Count); + + var first = evt.SlotMigrations[0]; + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 7000), first.Source); + Assert.Equal(new IPEndPoint(IPAddress.Loopback, 7001), first.Target); + Assert.Equal(new SlotRange(0, 99), Assert.Single(first.Slots)); + + var second = evt.SlotMigrations[1]; + Assert.Equal(2, second.Slots.Count); + Assert.Equal(new SlotRange(1000, 1000), second.Slots[0]); + Assert.Equal(new SlotRange(2000, 2500), second.Slots[1]); + Assert.Equal("1000,2000-2500", second.RawSlots); + } + } + + [Fact] + public async Task CapturedEnterpriseFramesAreUnderstood() + { + // Captured from a real Redis Cloud QA endpoint (Enterprise 8.6.2, OSS cluster API) during an actual + // slot migration on 2026-08-27. Byte for byte, except that the node addresses are rewritten into the + // private range - the lengths are preserved, so the $20 and $18 counts below are still the real ones: + // + // >3\r\n$10\r\nSMIGRATING\r\n:18\r\n$9\r\n8892-8991\r\n + // >3\r\n$9\r\nSMIGRATED\r\n:19\r\n*1\r\n*3\r\n$20\r\n10.129.228.140:13486\r\n + // $18\r\n10.252.90.18:13486\r\n$9\r\n8892-8991\r\n + // + // Things this pins, each of which was an assumption before: the type name arrives as a *bulk* string; + // the sequence number as a RESP integer rather than a string; neither cluster notification carries a + // time element; SMIGRATING's slots are a flat string; and SMIGRATED nests an array *of* triplets - one + // here - rather than a single flat triple. The fake emits this shape, so this test is the real frame. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, "8892-8991", sequenceId: 18); + + var migrating = await events.NextAsync(); + log.WriteLine(migrating.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrating, migrating.NotificationType); + Assert.Equal(18, migrating.SequenceId); + Assert.Null(migrating.Time); // no time element on the wire + Assert.Equal("8892-8991", migrating.Payload); + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [("10.129.228.140:13486", "10.252.90.18:13486", "8892-8991")], sequenceId: 19); + + var migrated = await events.NextAsync(); + log.WriteLine(migrated.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.SlotMigrated, migrated.NotificationType); + Assert.Equal(19, migrated.SequenceId); + Assert.Null(migrated.Time); + + var migration = Assert.Single(migrated.SlotMigrations); + Assert.Equal(new IPEndPoint(IPAddress.Parse("10.129.228.140"), 13486), migration.Source); + Assert.Equal(new IPEndPoint(IPAddress.Parse("10.252.90.18"), 13486), migration.Target); + Assert.Equal(new SlotRange(8892, 8991), Assert.Single(migration.Slots)); + } + } + + [Fact] + public async Task CapturedMovingFrameIsUnderstood() + { + // Captured from the same deployment during a maintenance_mode scenario, byte for byte: + // + // >4\r\n$6\r\nMOVING\r\n:0\r\n:15\r\n_\r\n + // + // Four elements: type, sequence *zero*, a 15-second window, and an explicit RESP3 null for the + // address - so "no replacement given, reconnect the way you connected". The proxy then closed the + // socket, which is the point of MOVING: you are told to move, and then the connection goes away. + // + // The sequence number is zero because this was the first event of that chain, not because MOVING is + // special - which is the point: zero is a legal value, so dedup tracks "have we seen one" as a separate + // bit rather than treating a stored zero as unset. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendMoving(null, timeSeconds: 15, newEndpoint: null, sequenceId: 0); + + var moving = await events.NextAsync(); + log.WriteLine(moving.RawMessage ?? "(none)"); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + Assert.Equal(0, moving.SequenceId); + Assert.Equal(TimeSpan.FromSeconds(15), moving.Time); + Assert.Null(moving.NewEndPoint); // the null is the documented "use what you already have" + Assert.Null(moving.Payload); + + // and the window it opened is what covers the reconnect that follows the socket closing + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "MOVING should have relaxed timeouts"); + } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating, MaintenanceNotificationKind.Migrated, "27")] + [InlineData(MaintenanceNotificationKind.FailingOver, MaintenanceNotificationKind.FailedOver, "21")] + public async Task CapturedShardNotificationsAreUnderstood( + MaintenanceNotificationKind opening, + MaintenanceNotificationKind closing, + string shard) + { + // Captured from the same deployment, byte for byte, running the fault injector's maintenance_mode and + // failover scenarios: + // + // >4\r\n$9\r\nMIGRATING\r\n:0\r\n:2\r\n$6\r\n["27"]\r\n + // >3\r\n$8\r\nMIGRATED\r\n:1\r\n$6\r\n["27"]\r\n + // >4\r\n$12\r\nFAILING_OVER\r\n:0\r\n:2\r\n$6\r\n["21"]\r\n + // >3\r\n$11\r\nFAILED_OVER\r\n:1\r\n$6\r\n["21"]\r\n + // + // Both families have the same shape, and it is the shape the parser assumed: the *opening* + // notification carries a time and the *closing* one has no time element at all - which is why + // CarriesTime is keyed on the notification type rather than on sniffing whether the third element + // looks like a number. The shard list is a stringified JSON array of id strings, quotes included, and + // is carried through opaquely: nothing a client does depends on which shard it was. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var shards = $"[\"{shard}\"]"; + server.SendShardNotification(null, opening, timeSeconds: 2, shardIds: shards, sequenceId: 0); + + var open = await events.NextAsync(); + log.WriteLine(open.RawMessage ?? "(none)"); + Assert.Equal(TimeSpan.FromSeconds(2), open.Time); + Assert.Equal(shards, open.Payload); + Assert.Equal(0, open.SequenceId); + + // note 2 seconds is what the server actually said, and is shorter than the relaxed timeout floor; + // the window is clamped up to the floor rather than being honoured literally + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, $"{opening} should have relaxed timeouts"); + + server.SendShardNotification(null, closing, timeSeconds: null, shardIds: shards, sequenceId: 1); + + var closed = await events.NextAsync(); + log.WriteLine(closed.RawMessage ?? "(none)"); + Assert.Null(closed.Time); // three elements on the wire: no time to read + Assert.Equal(shards, closed.Payload); + Assert.Equal(1, closed.SequenceId); + } + } + + [Fact] + public async Task OneLogicalEventFromEveryNodeRaisesOneEvent() + { + // Observed on the real deployment: every node broadcasts a given event, and all the copies carry the + // same sequence number - the id identifies the event, not the delivery. So a two-node cluster delivers + // one migration twice. Both deliveries have to be *acted on*, because the timeout relaxation is + // per-server, but the consumer wants one callback rather than one per proxy. + var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + GetHost(server.DefaultEndPoint, out var port); + var second = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + server.Migrate((RedisKey)"elsewhere", second); // owns something, so the client connects to it + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var events = new EventCollector(conn); + using (server) + { + var muxer = (IInternalConnectionMultiplexer)conn; + Assert.True( + await Poll.UntilAsync(() => + { + var endpoints = conn.GetEndPoints(); + return endpoints.Length == 2 && endpoints.All(ep => muxer.GetServerEndPoint(ep).MaintenanceNotificationsActive); + }), + "both nodes should be connected and opted in, or there is only one delivery and this proves nothing"); + + // one send, two clients: this is the fan-out, from the server side + var delivered = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + Assert.Equal(2, delivered); + + // both servers relax - that is the internal work that has to happen per connection... + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().All(ep => muxer.GetServerEndPoint(ep).IsMaintenanceRelaxed)), + "every node that was told should have relaxed its own timeouts"); + + // ...and the consumer sees it once + Assert.Equal(1, events.Count); + var evt = await events.NextAsync(); + Assert.Equal(5, evt.SequenceId); + log.WriteLine($"{events.Count} event(s) from {delivered} deliveries: {evt.RawMessage}"); + Assert.Contains(evt.EndPoint, conn.GetEndPoints()); // whichever told us first + } + } + + /// + /// The catch-up channel: Redis Enterprise retains the most recent shard-scoped *completion* and replays it + /// to each connection that opts in, so a client that connects after a disruption still learns it happened. + /// + /// + /// Observed on RS 8.0.22 (2026-08-28), arriving coalesced into the same TCP read as the +OK for the + /// opt-in, at ~16 ms. These tests key on the *cause* of that rather than the framing: the frame is flushed + /// the instant the opt-in is processed, and our opt-in is pipelined early, so a retained frame always + /// arrives while the connection is still handshaking. + /// + public class Retention(ITestOutputHelper log) + { + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( + ITestOutputHelper log, + Action beforeConnect, + NotificationLog? notifications = null) + { + var server = new InProcessTestServer(log); + beforeConnect(server); // the event happens *before* anybody connects: that is the whole point + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + config.LoggerFactory = notifications; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn, new EventCollector(conn)); + } + + /// + /// Captures the library's own "maintenance notification" log lines. + /// + /// + /// The only observable that works here. A retained frame arrives while the connection is still + /// handshaking, so a handler attached after ConnectAsync has already missed it - and the relaxed + /// window, which these tests used to key on, is deliberately *not* opened for a catch-up any more. A + /// logger can be attached through the configuration before connecting, so it sees everything, and it + /// asserts on the notification itself rather than on a side-effect of it. + /// + private sealed class NotificationLog : ILoggerFactory, ILogger + { + private readonly List _received = []; + + public IReadOnlyList Received + { + get { lock (_received) return _received.ToArray(); } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + if (message.Contains("Maintenance notification:", StringComparison.Ordinal)) + { + lock (_received) _received.Add(message); + } + } + + public void Dispose() { } + } + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrated)] + [InlineData(MaintenanceNotificationKind.FailedOver)] + public async Task RetainedCompletionIsReplayedToANewConnection(MaintenanceNotificationKind kind) + { + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => s.SendShardNotification(null, kind, timeSeconds: null, shardIds: "[\"27\"]", sequenceId: 5), + notifications); + + using (server) + await using (conn) + { + // The frame was received and understood - asserted from the library's own log, because the + // event collector is attached after connecting and has therefore already missed it. + var received = Assert.Single(notifications.Received); + log.WriteLine(received); + Assert.Contains(MaintenanceNotificationTypeFor(kind).ToString(), received); + Assert.Contains("seq=5", received); + Assert.Contains("(catch-up)", received); + + // ...and it did *not* relax anything. Measured on a live deployment: the same completion is + // replayed to fresh connections three hours after the event, and completions carry no + // time field, so relaxing here would mean every new connection to a database that had ever + // failed over began life patient about timeouts, and attributing any of them to maintenance + // that was long finished. + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.False(endpoint.IsMaintenanceRelaxed, "a catch-up completion should not open the post-event tail"); + Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); + + // ...and it is not reported to consumers either, which is the same decision applied + // consistently: we ignore it, so we do not hand somebody a notification they cannot date and + // therefore cannot act on correctly. The log line above is where it stays visible. + await events.AssertNoneAsync(); + + // ...and it must not have disturbed the handshake it arrived in the middle of, which is the + // other half of what this test is for: a push frame interleaved with our own handshake + // replies must be dispatched out-of-band rather than matched against one of them + Assert.True(conn.IsConnected); + Assert.True(await conn.GetDatabase().PingAsync() >= TimeSpan.Zero); + GC.KeepAlive(events); + } + } + + private static MaintenanceNotificationType MaintenanceNotificationTypeFor(MaintenanceNotificationKind kind) => kind switch + { + MaintenanceNotificationKind.Migrated => MaintenanceNotificationType.Migrated, + MaintenanceNotificationKind.FailedOver => MaintenanceNotificationType.FailedOver, + _ => MaintenanceNotificationType.None, + }; + + [Theory] + [InlineData(MaintenanceNotificationKind.Migrating)] + [InlineData(MaintenanceNotificationKind.FailingOver)] + [InlineData(MaintenanceNotificationKind.Moving)] + [InlineData(MaintenanceNotificationKind.SlotMigrating)] + [InlineData(MaintenanceNotificationKind.SlotMigrated)] + public async Task StartersAndSlotScopedEventsAreNotRetained(MaintenanceNotificationKind kind) + { + // The design property, asserted as a negative: the catch-up channel can only ever say "a + // disruption ended", never "one is starting". Nothing that demands action is replayed, so a + // reconnecting client cannot be told to move by a stale frame - which is what makes the MOVING + // handoff safe from replay by construction rather than by a guard. + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => + { + if (kind == MaintenanceNotificationKind.SlotMigrated) + { + s.SendSlotMigrations(null, kind, [("127.0.0.1:7000", "127.0.0.1:7001", "50-60")], sequenceId: 5); + } + else + { + s.SendShardNotification(null, kind, timeSeconds: 2, shardIds: "[\"27\"]", sequenceId: 5); + } + }, + notifications); + + using (server) + await using (conn) + { + await events.AssertNoneAsync(); + + // Asserted from the log rather than from the relaxed window: a catch-up no longer relaxes + // anything, so "nothing relaxed" would now be true whether the frame was retained or not, and + // this test would pass without exercising the property it exists for. + Assert.Empty(notifications.Received); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.False(endpoint.IsMaintenanceRelaxed, $"{kind} must not be retained, so nothing should have relaxed"); + } + } + + [Fact] + public async Task RetentionReplacesRatherThanAccumulates() + { + // "most recent completion", not a queue: a connection sees at most one of these however many + // events went past, which is why the client side needs no window or ring for the catch-up case + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => + { + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"1\"]", sequenceId: 5); + s.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, null, "[\"2\"]", sequenceId: 6); + }, + notifications); + + using (server) + await using (conn) + { + // one replay, and it is the later event - not both, and not the earlier one + var received = Assert.Single(notifications.Received); + log.WriteLine(received); + Assert.Contains(nameof(MaintenanceNotificationType.FailedOver), received); + Assert.Contains("seq=6", received); + await events.AssertNoneAsync(); // received, but not reported + } + } + + [Fact] + public async Task RetentionCanBeTurnedOff() + { + // not every deployment retains, and a test of the no-retention case should not have to reason + // about which notification kinds happen to be retained + var notifications = new NotificationLog(); + var (server, conn, events) = await ConnectAsync( + log, + s => + { + s.RetainCompletions = false; + s.SendShardNotification(null, MaintenanceNotificationKind.Migrated, null, "[\"27\"]", sequenceId: 5); + }, + notifications); + + using (server) + await using (conn) + { + await events.AssertNoneAsync(); + Assert.Empty(notifications.Received); // nothing arrived at all, which is the point + Assert.False(((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).IsMaintenanceRelaxed); + } + } + } + + [Theory] + [InlineData(null)] // the measured shape: the close arrives *after* the announced window + [InlineData(300)] // a less generous proxy: the close beats the window it announced + public async Task MovingClosesTheConnectionAndWeRecover(int? closeDelayMilliseconds) + { + // The defining half of MOVING, which no test covered until now: we are told to move, and then the + // socket goes away. Measured on RS (2026-08-28): the close landed at +18.4s and +16.6s against a + // declared 15s window, so the window is a floor with slack rather than a deadline - which is why the + // default case here lets the fake overshoot, and why nothing below asserts on the window expiring. + // The short case is the defensive one: a client that treats the announced window as guaranteed fails + // it. What must hold either way is that the connection comes back and the relaxed window opened by the + // notification is what covers the reconnect. + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.MovingClosesConnection = true; + server.MovingCloseDelay = closeDelayMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : null; + server.SendMoving(null, timeSeconds: 1, newEndpoint: null, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(endpoint.IsMaintenanceRelaxed, "the window has to be open before the socket dies"); + + // the client must reconnect on its own; nothing here asks it to. Note the probe has to tolerate + // throwing rather than returning false: a command issued in the window between the socket dying + // and the reconnect completing raises, and that is the state being polled *out* of + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 15_000), + "the multiplexer should have re-established itself after the close"); + + log.WriteLine($"close delay {closeDelayMilliseconds?.ToString() ?? "announced+slack"}: reconnected, relaxed = {endpoint.IsMaintenanceRelaxed}"); + } + } + + [Fact] + public async Task AHandoffThatMissesTheWindowSaysSo() + { + // The contract asks for the replacement to be *fully established* before the announced window runs + // out, and to report it when that is exceeded. Worth having because the two outcomes are otherwise + // indistinguishable from outside: commands succeed either way, since the relaxed window covers the + // gap, so a handoff that quietly took three times its budget looks exactly like one that worked. + // + // Provoked by making the *handshake* slow rather than the socket unreachable: the in-process transport + // routes by logical endpoint, so a named successor still connects here (which is why the sibling test + // checks intent rather than redirection). A reply-delaying server means the replacement connects but + // cannot finish establishing inside the window, which is precisely the case the warning is for. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + // Slowed *before* the notification, deliberately. Setting the latency afterwards races the + // handoff: the jitter on a short window can be almost nothing and an in-process reconnect takes + // about a millisecond, so the replacement can be established before the latency lands - a correct + // no-warning outcome, and a test that fails on a loaded machine. Measured: it did, once, in a + // two-core whole-suite run. + server.SetLatency(TimeSpan.FromSeconds(5)); + + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + server.SendMoving(null, timeSeconds: 2, newEndpoint: successor, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + Assert.True( + await Poll.UntilAsync( + () => logs.Lines.Any(l => l.Contains("did not establish a replacement", StringComparison.Ordinal)), + timeoutMilliseconds: 20_000), + "a handoff that misses the announced window should be reported as a warning"); + + var warning = logs.Lines.First(l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + log.WriteLine(warning); + Assert.Contains("2000ms", warning); // the window it was given, so the log says what was missed + + // let the server answer normally again, so teardown is not fighting the latency + server.SetLatency(TimeSpan.Zero); + } + } + + [Fact] + public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() + { + // The point of D6: today a MOVING is survivable because the socket eventually closes and we reconnect, + // but the announced window goes unused. Here nothing closes the connection - MovingClosesConnection is + // deliberately left off - so the *only* thing that can replace it is our own handoff. + // + // The named-successor branch is what is exercised, because the in-process transport has no socket and so + // no current address for the DNS branch to compare against; the deciding half of that branch is covered + // by MaintenanceHandoffTests against a supplied resolver. What this proves is the acting half: drain, + // drop, and come back. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + var optInsBefore = server.TotalMaintenanceOptIns; + Assert.Equal(1, optInsBefore); // the handshake's own opt-in, so the increment below means something + + // A handoff has to be *visible*: the replacement raises ConnectionRestored, so reporting nothing for + // the drop would leave a consumer tracking connection state with an unpaired restore. + var failures = new List(); + conn.ConnectionFailed += (_, e) => + { + lock (failures) failures.Add(e.FailureType); + }; + + // a *different* address, so the handoff target is distinguishable from where we already are + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + server.SendMoving(null, timeSeconds: 2, newEndpoint: successor, sequenceId: 0); + + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + Assert.Equal(successor, moving.NewEndPoint); + + // The handoff should have recorded where to go next. Note the in-process transport routes by + // *endpoint* rather than by socket address, so the fake cannot observe the redirection itself - + // that is what the live scenario test covers. What is checked here is the intent. + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True( + await Poll.UntilAsync(() => endpoint.HandoffTarget is not null || endpoint.HandoffRecycles > 0, timeoutMilliseconds: 10_000), + "the handoff should have taken the named successor"); + log.WriteLine($"handoff target: {endpoint.HandoffTarget?.ToString() ?? "(cleared after reconnect)"}; recycles={endpoint.HandoffRecycles}"); + + // a fresh connection re-sends the opt-in, which is how a recycle is visible from the server's side + Assert.True( + await Poll.UntilAsync(() => server.TotalMaintenanceOptIns > optInsBefore, timeoutMilliseconds: 15_000), + "the handoff should have replaced the connection without the server closing it"); + + log.WriteLine($"opt-ins: {optInsBefore} -> {server.TotalMaintenanceOptIns}"); + + Assert.True( + await Poll.UntilAsync( + () => + { + lock (failures) return failures.Contains(ConnectionFailureType.MaintenanceHandoff); + }, + timeoutMilliseconds: 5_000), + "the recycle should be reported as a MaintenanceHandoff rather than silently"); + + // ...and the deadline warning stays silent, which is the other half of AHandoffThatMissesTheWindow- + // SaysSo: a warning that fires on the path that worked is noise rather than a diagnostic. + Assert.DoesNotContain(logs.Lines, l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + + lock (failures) + { + log.WriteLine($"reported: {string.Join(", ", failures)}"); + + // and *only* as that: reporting it as a socket failure would put planned maintenance into + // everybody's fault dashboards + Assert.DoesNotContain(ConnectionFailureType.SocketFailure, failures); + Assert.DoesNotContain(ConnectionFailureType.SocketClosed, failures); + } + + // ...and the replacement is usable, which is the only outcome a caller cares about + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 15_000), + "the client should be serving commands on the replacement connection"); + } + } + + [Fact] + public async Task MalformedTripletIsSkippedNotFatal() + { + // one bad entry must not lose the migrations we could have applied - the same choice go-redis makes + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7000", "127.0.0.1:7001", "not-a-slot-list"), // kept, but with no parsed slots + ("127.0.0.1:7002", "?", "50-60"), // an unnameable target, kept with a null Target + ("127.0.0.1:7004", "127.0.0.1:7005", "70"), + ]); + + var evt = await events.NextAsync(); + Assert.Equal(3, evt.SlotMigrations.Count); + Assert.Empty(evt.SlotMigrations[0].Slots); + Assert.Equal("not-a-slot-list", evt.SlotMigrations[0].RawSlots); // raw form survives + Assert.Null(evt.SlotMigrations[1].Target); + Assert.Equal(new SlotRange(50, 60), Assert.Single(evt.SlotMigrations[1].Slots)); + Assert.Equal(new SlotRange(70, 70), Assert.Single(evt.SlotMigrations[2].Slots)); + } + } + + [Fact] + public async Task MissingSequenceIdStillReportsTheNotification() + { + // stricter-than-shipped-clients was the bug here: go-redis length-checks these at two elements and + // reads no sequence number at all, so dropping a frame for want of one loses a real disruption + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + // two elements, with an unreadable seq - the floor go-redis works to + Assert.True(server.SendRawPush(null, "MIGRATING", "not-a-number") > 0); + + var evt = await events.NextAsync(); + log.WriteLine(evt.RawMessage ?? "(no message)"); + Assert.Equal(MaintenanceNotificationType.Migrating, evt.NotificationType); + } + } + + [Theory] + [InlineData("NOT_A_REAL_KIND", "1", "5")] // a type we don't know + [InlineData("NOT_A_REAL_KIND")] // ...and one with nothing else at all + public async Task MalformedNotificationIsDroppedAndTheConnectionSurvives(params string[] parts) + { + // the important half of this feature's safety: an unparseable push frame must be consumed and + // forgotten. If it fell through to the command matcher it would steal the next command's reply, and + // everything after that would be answering the wrong question + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + Assert.True(server.SendRawPush(null, parts) > 0); + await events.AssertNoneAsync(); + + // the connection is not merely alive but *in sync*: each reply is the answer to its own command + var db = conn.GetDatabase(); + for (int i = 0; i < 10; i++) + { + await db.StringSetAsync($"maint-sync-{i}", i); + } + + for (int i = 0; i < 10; i++) + { + Assert.Equal(i, (int)await db.StringGetAsync($"maint-sync-{i}")); + } + } + } + + [Fact] + public async Task NotificationsArriveInterleavedWithCommands() + { + // a push can land at any point, including between a command and its reply; nothing may be lost or + // mismatched either way + var (server, conn, events) = await ConnectAsync(log); + using (server) + await using (conn) + { + var db = conn.GetDatabase(); + var pending = new List>(); + for (int i = 0; i < 50; i++) + { + await db.StringSetAsync($"maint-interleave-{i}", i); + pending.Add(db.StringGetAsync($"maint-interleave-{i}")); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: i); + } + + var values = await Task.WhenAll(pending); + Assert.Equal(Enumerable.Range(0, 50), values.Select(x => (int)x)); + + // and every notification arrived + for (int i = 0; i < 50; i++) + { + await events.NextAsync(); + } + + log.WriteLine($"{values.Length} commands and 50 notifications, all accounted for"); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs new file mode 100644 index 000000000..8c4fb6581 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInClientTests.cs @@ -0,0 +1,356 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// The client half of the opt-in: what we send during handshake, and what we make of the answer. The server +/// half is . +/// +[RunPerProtocol] +public class MaintenanceOptInClientTests(ITestOutputHelper log) +{ + private static InProcessTestServer CreateServer(ITestOutputHelper log) => new(log); + + private static ConfigurationOptions Config(InProcessTestServer server, MaintenanceNotificationMode mode) + { + var config = server.GetClientConfig(defaultOnly: true); + config.MaintenanceNotifications = mode; + return config; + } + + private static bool IsActive(IConnectionMultiplexer conn, InProcessTestServer server) + => ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint).MaintenanceNotificationsActive; + + private static List OptedIn(InProcessTestServer server) + { + var found = new List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } + + /// + /// Asserts that Enabled refused: either the connect threw, or it returned a connection that does + /// not stay usable. + /// + /// + /// Both outcomes are the same refusal, and which one a caller sees is a race: the reconcile records the + /// failure from the handshake-completion path, which can land either side of ConnectAsync deciding + /// it has a connection. Asserting only the throw made this flaky on a two-core runner, and asserting the + /// throw is not the point - not being left with a working connection is. + /// + private static async Task AssertRefusedAsync(ConfigurationOptions config, ITestOutputHelper log) + { + ConnectionMultiplexer? conn = null; + try + { + conn = await ConnectionMultiplexer.ConnectAsync(config); + } + catch (RedisConnectionException ex) + { + log.WriteLine($"refused at connect: {ex.Message}"); + return; + } + + await using (conn) + { + var endpoint = conn.GetEndPoints().Single(); + var unusable = await Poll.UntilAsync(() => !conn.GetServer(endpoint).IsConnected); + log.WriteLine($"connected, then unusable: {unusable}"); + Assert.True(unusable, "the connection should not have stayed usable"); + } + } + + [Fact] + public async Task HandshakeOptsInWhenAuto() + { + using var server = CreateServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Auto)); + + var resp3 = TestContext.Current.IsResp3(); + log.WriteLine($"protocol: {TestContext.Current.GetProtocol()}, opted in: {OptedIn(server).Count}"); + + // RESP3-only by design: under RESP2 the server could accept the request and then never deliver + // anything, so we don't ask - and under Auto that is simply the feature being off + Assert.Equal(resp3, IsActive(conn, server)); + Assert.Equal(resp3 ? 1 : 0, OptedIn(server).Count); + + if (resp3) + { + // The endpoint type now defaults to Auto, and over the in-process transport there is no socket + // address to classify - so it resolves to "none", meaning "send me no address, I will reconnect the + // way I connected". That is the honest answer where the scope cannot be determined, and it is what + // a tunnel or a Unix domain socket gets. Over a real socket this would be one of the four + // ip/fqdn forms; see MaintenanceEndpointTypeResolverTests. + Assert.Equal("none", Assert.Single(OptedIn(server)).MovingEndpointType); + } + + // ...and the connection is entirely usable either way + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task DisabledSendsNothing() + { + using var server = CreateServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Disabled)); + + Assert.Empty(OptedIn(server)); + Assert.False(IsActive(conn, server)); + } + + [Fact] + public async Task DefaultIsOff() + { + // the library default has to be Disabled: OSS Redis, Valkey and Garnet don't know the subcommand, and + // an unsolicited error reply on every connection would be a poor first impression + using var server = CreateServer(log); + var config = server.GetClientConfig(defaultOnly: true); // untouched, so whatever the provider says + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Equal(MaintenanceNotificationMode.Disabled, config.MaintenanceNotifications); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.UnknownSubcommand)] + [InlineData(MaintenanceNotificationSupport.Disabled)] + public async Task AutoToleratesAServerThatRefuses(MaintenanceNotificationSupport support) + { + // the whole point of Auto: ask, and carry on regardless. This is what lets the entire test suite run + // with the opt-in on against servers that will never send a notification + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(Config(server, MaintenanceNotificationMode.Auto)); + + Assert.False(IsActive(conn, server)); + Assert.Empty(OptedIn(server)); + Assert.True(conn.GetServer(server.DefaultEndPoint).IsConnected); + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task EnabledFailsAgainstAServerThatRefuses() + { + // Enabled is a requirement, not a preference: a connection that silently won't deliver notifications + // is not the connection that was asked for + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.MaintenanceNotifications = MaintenanceNotificationSupport.UnknownSubcommand; + + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task AutoIsOffWhenTheServerDowngradesToResp2() + { + // the case that makes this RESP3-only: the server accepts the opt-in and then answers HELLO as RESP2, + // so nothing would ever arrive. We asked, the server said OK, and we still treat it as off + using var server = CreateServer(log); + server.MaxProtocolVersion = RedisProtocol.Resp2; + var config = Config(server, MaintenanceNotificationMode.Auto); + config.Protocol = RedisProtocol.Resp3; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Single(OptedIn(server)); // the server took it... + Assert.False(IsActive(conn, server)); // ...and we know better than to believe it + Assert.Equal("value", await Set(conn)); + } + + [Fact] + public async Task EnabledFailsWhenTheServerDowngradesToResp2() + { + using var server = CreateServer(log); + server.MaxProtocolVersion = RedisProtocol.Resp2; + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.Protocol = RedisProtocol.Resp3; + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task EnabledFailsWhenResp2WasOurOwnChoice() + { + // no exemption for a contradiction we configured ourselves: requiring a RESP3-only feature over a + // RESP2 connection cannot be honoured, and half-honouring it silently is what Enabled exists to avoid + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.Protocol = RedisProtocol.Resp2; + config.AbortOnConnectFail = true; + config.ConnectRetry = 1; + + await AssertRefusedAsync(config, log); + } + + [Fact] + public async Task AutoIsHappyOnResp2() + { + // the counterpart: Auto is best-effort, so an explicit RESP2 just means the feature is off + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.Protocol = RedisProtocol.Resp2; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + Assert.Empty(OptedIn(server)); + Assert.False(IsActive(conn, server)); + Assert.Equal("value", await Set(conn)); + } + + /// + /// Captures log messages so a test can assert on what an operator would actually see. + /// + private sealed class CapturingLoggerFactory : ILoggerFactory, ILogger + { + public List Messages { get; } = []; + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (Messages) Messages.Add(formatter(state, exception)); + } + + public string All + { + get { lock (Messages) return string.Join("\n", Messages); } + } + + public void Dispose() { } + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.Supported, "Maintenance notifications accepted")] + [InlineData(MaintenanceNotificationSupport.Disabled, "Maintenance notifications refused")] + public async Task TheLogSaysWhetherTheFeatureIsLive(MaintenanceNotificationSupport support, string expected) + { + // This is the diagnostic docs/ServerMaintenanceEvent.md tells people to use, so it is worth a test. + // It also guards a mistake that was live for a while: the refusal was reported via + // PhysicalConnection.OnDetailLog, which is [Conditional("PARSE_DETAIL")] and compiles away in any normal + // build - so the reason a server declined was invisible to everybody who was not debugging the parser. + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + + var captured = new CapturingLoggerFactory(); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.LoggerFactory = captured; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + log.WriteLine(captured.All); + Assert.Contains(expected, captured.All); + } + + [Theory] + [InlineData(MaintenanceEndpointType.ServerDefault, null)] + [InlineData(MaintenanceEndpointType.InternalIp, "internal-ip")] + [InlineData(MaintenanceEndpointType.InternalFqdn, "internal-fqdn")] + [InlineData(MaintenanceEndpointType.ExternalIp, "external-ip")] + [InlineData(MaintenanceEndpointType.ExternalFqdn, "external-fqdn")] + [InlineData(MaintenanceEndpointType.None, "none")] + // Auto over the in-process transport: there is no socket to classify, so it resolves to "none" rather than + // guessing at a scope. Over a real socket to a loopback address it would derive internal-ip. + [InlineData(MaintenanceEndpointType.Auto, "none")] + public async Task MovingEndpointTypeIsSentWhenAskedFor(MaintenanceEndpointType type, string? expected) + { + // The point of asking: every MOVING observed on a real deployment carried no address, and every one of + // those was requested with a bare ON - so the working theory is that the server default amounts to + // "none". ServerDefault keeps that behaviour (send nothing); anything else says so explicitly. + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Enabled); + config.MaintenanceMovingEndpointType = type; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(IsActive(conn, server), "the server should have accepted the opt-in"); + + var client = Assert.Single(OptedIn(server)); + log.WriteLine($"{type} -> moving-endpoint-type: {client.MovingEndpointType ?? "(none sent)"}"); + Assert.Equal(expected, client.MovingEndpointType); + } + + [Fact] + public async Task UnsupportedMovingEndpointTypeIsRefusedNotFatal() + { + // A server that does not know a type answers with an error, and that is a refusal like any other: with + // Auto we carry on without the feature rather than failing the connection. + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + server.SupportedMovingEndpointTypes = ["external-fqdn"]; // this deployment offers one form only + + var config = Config(server, MaintenanceNotificationMode.Auto); + config.MaintenanceMovingEndpointType = MaintenanceEndpointType.InternalIp; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.False(IsActive(conn, server), "an unsupported endpoint type is a refusal"); + Assert.Equal("value", await Set(conn)); // ...and the connection is still perfectly usable + } + + [Fact] + public async Task OptInIsReArmedOnReconnect() + { + // per-connection state, so a reconnect that didn't re-send it would leave us silently unsubscribed + Assert.SkipUnless(TestContext.Current.IsResp3(), "the opt-in is only sent under RESP3"); + + using var server = CreateServer(log); + var config = Config(server, MaintenanceNotificationMode.Auto); + config.AllowSimulateConnectionFailure = true; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + Assert.True(IsActive(conn, server)); + + var before = server.TotalMaintenanceOptIns; + conn.GetServer(server.DefaultEndPoint).SimulateConnectionFailure(SimulatedFailureType.All); + await UntilCondition(() => server.TotalMaintenanceOptIns > before); + + log.WriteLine($"opt-ins: {before} -> {server.TotalMaintenanceOptIns}"); + Assert.True(server.TotalMaintenanceOptIns > before, "the opt-in should be sent again on the new connection"); + + // ...and then wait for *our* side of it. The count above is the server having processed the opt-in, + // whereas IsActive is us having processed its reply - a beat later, so asserting it directly is a race + // that only shows up when the runner is starved of cores. + await UntilCondition(() => IsActive(conn, server)); + Assert.True(IsActive(conn, server), "the feature should be live again on the replacement connection"); + } + + private static async Task UntilCondition(System.Func condition, int timeoutMilliseconds = 5000) + { + for (int i = 0; i < timeoutMilliseconds / 50 && !condition(); i++) + { + await Task.Delay(50); + } + } + + private static async Task Set(IConnectionMultiplexer conn) + { + var db = conn.GetDatabase(); + await db.StringSetAsync("maint-optin", "value"); + return await db.StringGetAsync("maint-optin"); + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs new file mode 100644 index 000000000..dcec5dcac --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceOptInServerTests.cs @@ -0,0 +1,159 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// The server half of the maintenance-notification contract, exercised directly. The client does not opt in +/// yet, so these drive the command as a caller would - which is also how any other test will be able to opt +/// in once the client does, since this is ordinary server functionality rather than a special test server. +/// +public class MaintenanceOptInServerTests(ITestOutputHelper log) +{ + private static InProcessTestServer CreateServer(ITestOutputHelper log) => new(log); + + private static async Task OptInAsync(IConnectionMultiplexer conn, InProcessTestServer server, params object[] args) + => await conn.GetServer(server.DefaultEndPoint).ExecuteAsync("client", args.Prepend("maint_notifications").ToArray()); + + [Fact] + public async Task BareOnIsAcceptedAndRecorded() + { + // "CLIENT MAINT_NOTIFICATIONS ON" with no parameters is explicitly valid, and means "server defaults" + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + Assert.Equal("OK", (string?)await OptInAsync(conn, server, "on")); + + var client = Assert.Single(OptedIn(server)); + Assert.Null(client.MovingEndpointType); // server defaults, not a value we invented + Assert.Equal(1, client.MaintenanceNotificationOptInCount); + } + + [Theory] + [InlineData("internal-ip")] + [InlineData("internal-fqdn")] + [InlineData("external-ip")] + [InlineData("external-fqdn")] + [InlineData("none")] + public async Task EveryDefinedEndpointTypeIsAccepted(string endpointType) + { + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + Assert.Equal("OK", (string?)await OptInAsync(conn, server, "on", "moving-endpoint-type", endpointType)); + Assert.Equal(endpointType, Assert.Single(OptedIn(server)).MovingEndpointType); + } + + [Fact] + public async Task OffClearsTheOptIn() + { + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + await OptInAsync(conn, server, "on", "moving-endpoint-type", "external-fqdn"); + Assert.Single(OptedIn(server)); + + Assert.Equal("OK", (string?)await OptInAsync(conn, server, "off")); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData("sideways")] // not on/off + [InlineData("on", "moving-endpoint-type", "sideways")] // undefined endpoint type + [InlineData("on", "not-a-parameter", "value")] // unknown parameter + [InlineData("on", "moving-endpoint-type")] // parameter with no value + public async Task MalformedOptInIsRejected(params string[] args) + { + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + var ex = await Assert.ThrowsAsync( + async () => await OptInAsync(conn, server, args.Cast().ToArray())); + log.WriteLine(ex.Message); + Assert.Empty(OptedIn(server)); + } + + [Theory] + [InlineData(MaintenanceNotificationSupport.UnknownSubcommand)] + [InlineData(MaintenanceNotificationSupport.Disabled)] + public async Task UnsupportingServerRejectsTheOptIn(MaintenanceNotificationSupport support) + { + // the two ways a real server refuses: it has never heard of the subcommand (OSS, Valkey, Garnet), or + // it knows it and has the feature flag off. A client has to survive both + using var server = CreateServer(log); + server.MaintenanceNotifications = support; + await using var conn = await server.ConnectAsync(defaultOnly: true); + + var ex = await Assert.ThrowsAsync( + async () => await OptInAsync(conn, server, "on")); + log.WriteLine($"{support}: {ex.Message}"); + Assert.Empty(OptedIn(server)); + } + + [Fact] + public async Task NotificationsGoOnlyToConnectionsThatOptedIn() + { + // a real server sends to the connections that asked; sending to everything would let a client pass a + // test it should fail, by receiving notifications it never subscribed to + using var server = CreateServer(log); + await using var optedIn = await server.ConnectAsync(defaultOnly: true); + await using var notOptedIn = await server.ConnectAsync(defaultOnly: true); + + await OptInAsync(optedIn, server, "on", "moving-endpoint-type", "external-fqdn"); + var subscribed = OptedIn(server).Count(); + log.WriteLine($"{subscribed} of {server.ClientCount} connections opted in"); + + var sent = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + Assert.Equal(subscribed, sent); + Assert.NotEqual(server.ClientCount, sent); + } + + [Fact] + public async Task SequenceIdsAdvanceAndCanBeRepeated() + { + // the contract never defines these, so a client's use of them is its own invention - which means being + // able to repeat one deliberately is part of what the fake owes us + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + await OptInAsync(conn, server, "on"); + + var first = server.NextMaintenanceSequenceId; + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, 5)); + Assert.Equal(first + 1, server.NextMaintenanceSequenceId); + + // and an explicit id does not advance the counter, so a replay stays a replay + Assert.Equal(1, server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, 5, sequenceId: first)); + Assert.Equal(first + 1, server.NextMaintenanceSequenceId); + } + + [Fact] + public async Task RawPushIsNotGatedByOptIn() + { + // the malformed-payload hook, and the contrast is the point: with nobody opted in, a notification + // reaches no one while a raw push still lands. Asserted as gated-versus-not rather than against + // ClientCount, which is read at a different instant - under RESP2 the subscription connection can + // register in between, so comparing totals is a race rather than a property + using var server = CreateServer(log); + await using var conn = await server.ConnectAsync(defaultOnly: true); + + var gated = server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + var ungated = server.SendRawPush(null, "NOT_A_REAL_KIND", "nonsense"); + log.WriteLine($"notification reached {gated}, raw push reached {ungated}"); + + Assert.Equal(0, gated); + Assert.True(ungated > 0, "a raw push should reach connections that never opted in"); + } + + private static System.Collections.Generic.IEnumerable OptedIn(InProcessTestServer server) + { + var found = new System.Collections.Generic.List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs new file mode 100644 index 000000000..293ced8f8 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationEvidenceTests.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Does relaxation actually save a command, and does a timeout inside a window say why? +/// +/// +/// The gap these close: everything about relaxed timeouts was implemented and unit-tested, but nothing had ever +/// observed one *rescue* a command - four scenario runs against a real deployment produced zero command +/// failures, which is the right product outcome and useless as evidence. +/// +/// The obvious way to force failures - the fault injector's network_latency - turned out to be the wrong +/// tool. It applies netem to a whole node's interface, so it delays the cluster's own traffic and its DNS along +/// with the client's; 200ms across two of three nodes took a working deployment offline, and its +/// duration_seconds did not revert. A per-reply delay in the fake is precise, instant, and cannot break +/// anything - and it makes the difference measurable rather than anecdotal. +/// +/// +public class MaintenanceRelaxationEvidenceTests(ITestOutputHelper log) +{ + // The delays have to clear the heartbeat, not just the timeout: async timeouts are raised by the bridge + // heartbeat sweep on roughly a one-second cadence, not at the instant the deadline passes. A 600ms delay + // against a 200ms timeout therefore completes *successfully*, which is how the first version of this test + // found its control case passing when it should have failed. + private const int NormalTimeoutMs = 200, RelaxedTimeoutSeconds = 8, SlowReplySeconds = 3; + + private static async Task ConnectAsync(InProcessTestServer server) + { + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + config.SyncTimeout = NormalTimeoutMs; + config.AsyncTimeout = NormalTimeoutMs; + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(RelaxedTimeoutSeconds); + + return await ConnectionMultiplexer.ConnectAsync(config); + } + + private static List OptedIn(InProcessTestServer server) + { + var found = new List(); + server.ForAllClients(c => + { + if (c.MaintenanceNotifications) found.Add(c); + }); + return found; + } + + [Fact] + public async Task ARelaxedWindowRescuesACommandThatWouldOtherwiseTimeOut() + { + // Two clients, one server, one delay - and the notification sent to only one of them. That is the whole + // experiment: identical conditions, simultaneously, differing only in whether a disruption was + // announced. Relaxation is per-server-per-multiplexer, so the client that was never told keeps its + // ordinary timeout. + // + // Deliberately not "fail, then succeed" on a single connection: a timed-out command disrupts the + // connection, and with a slow reply still configured the reconnect handshake cannot complete, so the + // second half then fails in the backlog for an entirely unrelated reason. That is what the first + // version of this test did. + using var server = new InProcessTestServer(log); + + await using var told = await ConnectAsync(server); + var toldClient = Assert.Single(OptedIn(server)); + + await using var untold = await ConnectAsync(server); + Assert.Equal(2, OptedIn(server).Count); + + await told.GetDatabase().PingAsync(); + await untold.GetDatabase().PingAsync(); // both healthy before anything is slowed + + // well past the normal timeout *and* the heartbeat, comfortably inside the relaxed one + server.ResponseDelay = TimeSpan.FromSeconds(SlowReplySeconds); + + // announced to one connection only + server.SendShardNotification(toldClient, MaintenanceNotificationKind.Migrating, timeSeconds: 20, shardIds: "[\"1\"]", sequenceId: 0); + var endpoint = ((IInternalConnectionMultiplexer)told).GetServerEndPoint(server.DefaultEndPoint); + Assert.True( + await Poll.UntilAsync(() => endpoint.IsMaintenanceRelaxed, timeoutMilliseconds: 10_000), + "the notification should have relaxed timeouts on the connection that received it"); + + var rescued = told.GetDatabase().PingAsync(); + var doomed = untold.GetDatabase().PingAsync(); + + var rtt = await rescued; + log.WriteLine($"told about the disruption: succeeded after {rtt.TotalMilliseconds:0}ms"); + Assert.True(rtt.TotalMilliseconds > NormalTimeoutMs, "the command should have taken longer than the normal timeout allows"); + + var failure = await Assert.ThrowsAnyAsync(() => doomed); + log.WriteLine($"not told: {failure.GetType().Name}"); + Assert.True(failure is RedisTimeoutException or TimeoutException, $"expected a timeout, got {failure}"); + } + + [Fact] + public async Task ATimeoutInsideAWindowSaysWhichEventCausedIt() + { + using var server = new InProcessTestServer(log); + await using var conn = await ConnectAsync(server); + { + var db = conn.GetDatabase(); + await db.PingAsync(); + + // A long window on purpose. The attribution is read when the timeout is *raised*, so a window that + // has already expired attributes nothing - even though its disruption is what caused the delay. + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 20, shardIds: "[\"7\"]", sequenceId: 0); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + Assert.True(await Poll.UntilAsync(() => endpoint.IsMaintenanceRelaxed, timeoutMilliseconds: 5_000)); + + // beyond even the relaxed timeout, so the command fails *during* an announced disruption + server.ResponseDelay = TimeSpan.FromSeconds(RelaxedTimeoutSeconds * 3); + + var failure = await Assert.ThrowsAnyAsync(() => db.PingAsync()); + log.WriteLine($"{failure.GetType().Name}: {failure.Message}"); + + // This is what the attribution is for: "the deployment was failing over" rather than "your query + // is slow". Nothing had ever observed it fire, because no real run ever failed a command. + var maintenanceType = failure switch + { + RedisTimeoutException timeout => timeout.MaintenanceType, + RedisConnectionException connection => connection.MaintenanceType, + _ => MaintenanceNotificationType.None, + }; + + log.WriteLine($"attributed to: {maintenanceType}"); + Assert.Equal(MaintenanceNotificationType.FailingOver, maintenanceType); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs new file mode 100644 index 000000000..a98c0c058 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceRelaxationTests.cs @@ -0,0 +1,421 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Timeout relaxation: a server that announces a disruption gets its command timeouts raised for the duration, +/// as a floor and never a reduction. The window arithmetic is asserted through internals rather than by waiting +/// out real seconds - the interesting properties are "does it open, extend, close, and expire correctly", and a +/// test that sleeps for 30s to watch a cap expire is a test nobody runs. +/// +public class MaintenanceRelaxationTests(ITestOutputHelper log) +{ + private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + Action? configure = null) + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + configure?.Invoke(config); + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + private static ServerEndPoint Endpoint(IConnectionMultiplexer conn, InProcessTestServer server) + => ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + + /// + /// The notification arrives on the connection's read loop, so a test that sends one has to let it land. + /// + private static async Task UntilRelaxedAsync(ServerEndPoint endpoint, bool expected, int timeoutMilliseconds = 5000) + { + for (int i = 0; i < timeoutMilliseconds / 25 && endpoint.IsMaintenanceRelaxed != expected; i++) + { + await Task.Delay(25); + } + return endpoint.IsMaintenanceRelaxed; + } + + [Fact] + public async Task NoWindowMeansNoChange() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + Assert.False(endpoint.IsMaintenanceRelaxed); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + } + } + + [Fact] + public async Task OpeningNotificationRelaxesTimeouts() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + + Assert.True(await UntilRelaxedAsync(endpoint, true), "should be relaxed"); + + // 10s relaxed against a 5s configured timeout + Assert.Equal(10_000, endpoint.GetEffectiveTimeoutMilliseconds(5_000)); + } + } + + [Fact] + public async Task RelaxationIsAFloorAndNeverAReduction() + { + // a caller with a generous timeout keeps it; this is explicit in the contract + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.Equal(60_000, endpoint.GetEffectiveTimeoutMilliseconds(60_000)); + } + } + + [Fact] + public async Task ClosingNotificationLeavesThePostEventTail() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + await Task.Delay(250); // let it land; the tail means "still relaxed", so there is no flag to await + + // the server said it finished, and we are *still* relaxed - because that is when every other + // client that got the same notification comes back + Assert.True(endpoint.IsMaintenanceRelaxed, "the post-event tail should still be running"); + Assert.Equal(10_000, endpoint.GetEffectiveTimeoutMilliseconds(5_000)); + } + } + + [Fact] + public async Task AFaultIsStillAttributedToAWindowThatHasJustClosed() + { + // Async timeouts are raised by the bridge heartbeat, not at the deadline, so a command that timed out + // was already outstanding for its whole timeout before anybody looked - and a window covering its + // entire life can have closed by the time the exception is built. Reading the *active* type then + // reports None for a timeout maintenance plainly caused, which is the opposite of the point. + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 1); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false), "the window should have closed"); + + // closed, so nothing is relaxed any more... + Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + + // ...but a command with a 30s timeout could easily have spanned it, so a fault is still its doing + Assert.Equal(MaintenanceNotificationType.Migrated, endpoint.GetMaintenanceTypeForFault(30_000)); + log.WriteLine($"attributed to {endpoint.GetMaintenanceTypeForFault(30_000)} after the window closed"); + } + } + + [Fact] + public async Task AFaultIsNotAttributedToALongClosedWindow() + { + // The other half: the lookback is bounded by how long the command could have been waiting, so a + // short-timeout command that fails well after a window closed is not blamed on it. Without a bound + // this would attribute every timeout for the rest of the process to the last maintenance event. + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 1); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false)); + + // past the one-second floor that covers the heartbeat's own imprecision + await Task.Delay(1500); + Assert.Equal(MaintenanceNotificationType.None, endpoint.GetMaintenanceTypeForFault(100)); + + // ...while a command that *could* have been outstanding that long still is attributed + Assert.Equal(MaintenanceNotificationType.Migrated, endpoint.GetMaintenanceTypeForFault(30_000)); + } + } + + [Fact] + public async Task ACatchUpCompletionOpensNoTail() + { + // Measured on a live deployment: a completion is retained and replayed to whoever opts in next, with + // no age limit found (the same FAILED_OVER came back three hours later), and completions carry + // no time field - so nothing in the frame says how old it is. Relaxing on one would mean every new + // connection to a database that had ever failed over started life patient about timeouts, and + // reporting any that expired as caused by maintenance long since finished. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + endpoint.OnMaintenanceWindowClosed(MaintenanceNotificationType.FailedOver, sequenceId: 41, isCatchUp: true); + + Assert.False(endpoint.IsMaintenanceRelaxed, "a catch-up completion should not relax anything"); + Assert.Equal(MaintenanceNotificationType.None, endpoint.ActiveMaintenanceType); + Assert.Equal(1234, endpoint.GetEffectiveTimeoutMilliseconds(1234)); + } + } + + [Fact] + public async Task ACatchUpCompletionDoesNotCancelALiveWindow() + { + // The reason the catch-up path declines to *open* a window rather than closing one. Relaxation belongs + // to the ServerEndPoint and is shared by both bridges, so a subscription bridge that reconnects + // mid-disruption - and is handed the retained completion of some *earlier* event on the way in - must + // not cancel the window the live notification opened on the interactive bridge. Driven through the + // internals because the transport-level version of this needs one bridge to reconnect while another + // stays up, which is a lot of scaffolding to assert a state-machine rule. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 60); + Assert.True(await UntilRelaxedAsync(endpoint, true), "the live notification should have opened a window"); + + // an unrelated, older completion arrives as catch-up on a reconnecting bridge + endpoint.OnMaintenanceWindowClosed(MaintenanceNotificationType.Migrated, sequenceId: 7, isCatchUp: true); + + Assert.True(endpoint.IsMaintenanceRelaxed, "the live window should have survived the catch-up"); + Assert.Equal(MaintenanceNotificationType.FailingOver, endpoint.ActiveMaintenanceType); + } + } + + [Fact] + public async Task ClosingNotificationEndsItWhenThereIsNoTail() + { + var (server, conn) = await ConnectAsync(log, config => config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + Assert.False(await UntilRelaxedAsync(endpoint, false), "should have ended"); + } + } + + [Fact] + public async Task WindowExpiresOnItsOwn() + { + // the cap and the ordinary deadline share a mechanism; a one-second floor lets us watch it expire + // without a test that sleeps for half a minute + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero; + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + + // asks for 20s, capped to 1s + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.False(await UntilRelaxedAsync(endpoint, false), "the cap should have ended it"); + } + } + + [Fact] + public async Task NoTailAfterACapExpiry() + { + // if the window ended on the cap we never learned the event finished, so extending past the backstop + // would defeat the backstop + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.FromSeconds(30); // would be very visible + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + Assert.False(await UntilRelaxedAsync(endpoint, false), "no tail should follow a capped window"); + } + } + + [Fact] + public async Task ShorterNotificationDoesNotCutAnOpenWindowShort() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + + // a long window, then a short one on top: the short one must not shorten it + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 25); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 1); + await Task.Delay(1500); + + Assert.True(endpoint.IsMaintenanceRelaxed, "the longer window should still be in force"); + } + } + + [Fact] + public async Task ReplayedSequenceIdIsIgnored() + { + // our own invention, so kept conservative: an id we have already acted on cannot extend a window + var (server, conn) = await ConnectAsync(log, config => + { + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(1); + config.MaintenancePostEventRelaxedDuration = TimeSpan.Zero; + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + var seq = server.NextMaintenanceSequenceId; + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + Assert.False(await UntilRelaxedAsync(endpoint, false)); + + // replaying the same id must not reopen anything + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20, sequenceId: seq); + await Task.Delay(250); + Assert.False(endpoint.IsMaintenanceRelaxed, "a replayed id should not reopen the window"); + + // ...but a genuinely new one still does + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true), "a new id should still work"); + } + } + + [Fact] + public async Task RelaxationDoesNotSuppressConnectionFailure() + { + // the regression that would matter most: relaxation must touch command timeouts only. If it reached + // keep-alive or failure detection, a server that died mid-maintenance would linger for the window - + // turning a latency mitigation into an availability regression + var (server, conn) = await ConnectAsync(log, config => + { + config.AllowSimulateConnectionFailure = true; + config.MaintenanceRelaxedTimeout = TimeSpan.FromMinutes(5); // absurdly generous, deliberately + config.MaintenanceRelaxedWindowMax = TimeSpan.FromMinutes(5); + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 300); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + var failures = 0; + conn.ConnectionFailed += (_, _) => Interlocked.Increment(ref failures); + conn.GetServer(server.DefaultEndPoint).SimulateConnectionFailure(SimulatedFailureType.All); + + for (int i = 0; i < 100 && Volatile.Read(ref failures) == 0; i++) + { + await Task.Delay(25); + } + + log.WriteLine($"connection failures observed: {Volatile.Read(ref failures)}"); + Assert.True(Volatile.Read(ref failures) > 0, "a dead connection must still be noticed during a relaxed window"); + } + } + + [Theory] + [InlineData(true, MaintenanceNotificationType.Migrating)] + [InlineData(false, MaintenanceNotificationType.None)] + public async Task TimeoutReportsWhetherMaintenanceWasInForce(bool announce, MaintenanceNotificationType expected) + { + // "timeout" and "timeout during an announced migration" call for very different reactions from + // whoever reads the log, so the fault says which it was + var (server, conn) = await ConnectAsync(log, config => + { + config.AsyncTimeout = 200; + config.SyncTimeout = 200; + config.MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1); + config.MaintenanceRelaxedWindowMax = TimeSpan.FromSeconds(30); + config.HeartbeatInterval = TimeSpan.FromMilliseconds(100); + }); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + if (announce) + { + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + } + + server.SetLatency(TimeSpan.FromSeconds(5)); // longer than any timeout in play + var ex = await Assert.ThrowsAsync(() => conn.GetDatabase().StringGetAsync("maint-fault")); + log.WriteLine($"{ex.MaintenanceType}: {ex.Message}"); + Assert.Equal(expected, ex.MaintenanceType); + + // and it reaches the circuit-breaker/retry surface the same way + var context = new Availability.FaultContext(ex); + Assert.Equal(expected, context.MaintenanceType); + + server.SetLatency(TimeSpan.Zero); + } + } + + [Fact] + public async Task CommandsStillWorkThroughAWindow() + { + // relaxation is invisible to anything that is not timing out + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var endpoint = Endpoint(conn, server); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 20); + Assert.True(await UntilRelaxedAsync(endpoint, true)); + + var db = conn.GetDatabase(); + for (int i = 0; i < 20; i++) + { + await db.StringSetAsync($"relax-{i}", i); + } + for (int i = 0; i < 20; i++) + { + Assert.Equal(i, (int)await db.StringGetAsync($"relax-{i}")); + } + + // including the synchronous path, which has its own re-wait loop + Assert.Equal(7, (int)db.StringGet("relax-7")); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs new file mode 100644 index 000000000..5cddbb1eb --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MaintenanceTopologyRefreshTests.cs @@ -0,0 +1,400 @@ +using System; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Server; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Non-parallel deliberately. Several of these wait out a jittered refresh, and the retirement one depends on +/// the doomed server being *idle* - and IsIdle counts outstanding work, which a heartbeat ping in flight +/// supplies. On a loaded machine those complete slowly enough that pruning is starved, which is a real property +/// of the policy rather than a flaw in the test: see the design notes on why a usage-based grace rule was +/// dropped for exactly this reason. +/// +/// Reacting to a completed slot migration by re-reading the topology, rather than waiting to be told by a +/// -MOVED. Asserted from the server's side - did another CLUSTER command actually arrive - because +/// that is the thing the fleet pays for, and the thing a scoping bug would multiply. +/// +/// +[Collection(NonParallelCollection.Name)] +public class MaintenanceTopologyRefreshTests(ITestOutputHelper log) +{ + /// + /// Counts inbound CLUSTER commands, so a test can tell a refresh happened from the outside. + /// + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands, _subscribeCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public int SubscribeCommands => Volatile.Read(ref _subscribeCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0) + { + var command = request.GetString(0); + if (string.Equals(command, "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + else if (string.Equals(command, "ssubscribe", StringComparison.OrdinalIgnoreCase) + || string.Equals(command, "subscribe", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _subscribeCommands); + } + } + + return base.Execute(client, in request); + } + } + + private static async Task<(CountingServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + Action? configure = null, + MaintenanceNotificationMode mode = MaintenanceNotificationMode.Enabled) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + configure?.Invoke(server); + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = mode; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + /// + /// The jitter is up to a second, so a refresh is not immediate by design. + /// + private static Task UntilRefreshedAsync(CountingServer server, int before) + => Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 5000); + + [Fact] + public async Task SlotsMovingAwayFromUsTriggersARefresh() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", "0-99"), + ]); + + Assert.True(await UntilRefreshedAsync(server, before), "a refresh should follow slots leaving us"); + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + } + } + + [Fact] + public async Task SlotsMovingBetweenOtherNodesDoesNotTriggerARefresh() + { + // the whole herd argument: every node reports the same movements, so most notifications are about + // somebody else. Refreshing on those means every client in the fleet re-reads topology whenever any + // shard moves anywhere + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ("127.0.0.1:7100", "127.0.0.1:7101", "0-99"), + ("127.0.0.1:7102", "127.0.0.1:7103", "100-199"), + ]); + + await Task.Delay(2000); // longer than the jitter, so "not yet" cannot be mistaken for "never" + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task SourceIsMatchedByAnnouncedIdentityNotJustAddress() + { + // A node answers to its address *and* its announced hostname, and the delta may name either, so the + // scoping test resolves through the identity map rather than comparing endpoints as text. Announced + // before connecting, because the handshake's own topology read is what registers the identity - a + // hostname set afterwards is not known to us until something re-reads it. + const string Hostname = "node-1.redis.example.com"; + var (server, conn) = await ConnectAsync(log, s => s.SetHostname(s.DefaultEndPoint, Hostname)); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"{Hostname}:{port}", $"127.0.0.1:{port + 1}", "0-99"), + ]); + + Assert.True(await UntilRefreshedAsync(server, before), "the hostname form should resolve to us"); + } + } + + [Fact] + public async Task ProxyStyleCompletionDoesNotTriggerARefresh() + { + // MIGRATED/FAILED_OVER arrive in proxied deployments addressed as a single endpoint, where there is no + // topology for a refresh to learn - so they get relaxation and nothing else + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + + server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 5); + server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: 0); + + await Task.Delay(2000); + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task ShardedSubscriptionOnAMovedSlotIsReEstablished() + { + // mostly belt-and-braces: the server also sends an unsolicited SUNSUBSCRIBE for this, which we already + // act on. What this adds is being pre-emptive when SMIGRATED lands first, and covering the case where + // the unsubscribe never arrives - where the only other symptom is messages silently stopping + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("resub-channel"); + await sub.SubscribeAsync(channel, (_, _) => { }); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + log.WriteLine($"channel hashes to slot {slot}"); + + var before = server.SubscribeCommands; + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{slot}"), + ]); + + Assert.True( + await Poll.UntilAsync(() => server.SubscribeCommands > before), + "the sharded subscription should have been re-established"); + } + } + + [Fact] + public async Task ShardedSubscriptionOnAnUnaffectedSlotIsLeftAlone() + { + // knowing *which* slots moved is the advantage over the unsolicited-unsubscribe path: only the + // affected channels are touched, rather than everything subscribed on this server + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("untouched-channel"); + await sub.SubscribeAsync(channel, (_, _) => { }); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + var otherSlot = slot == 0 ? 1 : 0; + var before = server.SubscribeCommands; + + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{otherSlot}"), + ]); + + await Task.Delay(2000); + log.WriteLine($"subscribe commands: {before} -> {server.SubscribeCommands} (slot {slot} vs moved {otherSlot})"); + Assert.Equal(before, server.SubscribeCommands); + } + } + + [Fact] + public async Task OrdinaryPubSubIsNotSlotBoundAndIsLeftAlone() + { + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var sub = conn.GetSubscriber(); + await sub.SubscribeAsync(RedisChannel.Literal("plain-channel"), (_, _) => { }); + + var before = server.SubscribeCommands; + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", "0-16383"), // everything moves + ]); + + await Task.Delay(2000); + log.WriteLine($"subscribe commands: {before} -> {server.SubscribeCommands}"); + Assert.Equal(before, server.SubscribeCommands); + } + } + + [Theory] + [InlineData(MaintenanceNotificationMode.Disabled)] // only the unsolicited SUNSUBSCRIBE path can act + [InlineData(MaintenanceNotificationMode.Enabled)] // both paths see the same migration + public async Task RealMigrationRecoversTheSubscriptionWithoutStorming(MaintenanceNotificationMode mode) + { + // Until the fake announced its own migrations this could not be tested at all, and it is the case that + // matters: a real slot migration produces *both* an unsolicited SUNSUBSCRIBE (ordinary cluster + // behaviour, sent to every subscriber) and SMIGRATED (only to clients that opted in). Both paths lead + // to ResubscribeToServer, so the question is whether the subscription ends up established exactly + // once rather than twice or not at all. + var (server, conn) = await ConnectAsync(log, mode: mode); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var doomed = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Sharded("migrating-channel"); + var received = 0; + await sub.SubscribeAsync(channel, (_, _) => Interlocked.Increment(ref received)); + + var slot = ((ConnectionMultiplexer)conn).ServerSelectionStrategy.HashSlot(channel); + var before = server.SubscribeCommands; + + server.NotifyOnMigrate = true; + server.Migrate(slot, doomed); // the whole realistic sequence, from one call + + Assert.True(await Poll.UntilAsync(() => server.SubscribeCommands > before), "should have resubscribed"); + await Task.Delay(2000); // and settle, so a second resubscribe would show up + + var resubscribes = server.SubscribeCommands - before; + log.WriteLine($"{mode}: {resubscribes} (re)subscribe command(s) after a real migration"); + + // The property is boundedness, not an absolute count: the unsolicited-unsubscribe path has its own + // pre-existing retry-and-redirect behaviour. Measured at 4 with notifications off and 5 with them + // on - the extra one being the fallback acting on a subscription that path left attached to + // nothing, which is the feature working. What must not happen is one attempt per notification. + // Bounded, not exact: the unsolicited-unsubscribe path retries and follows redirects, and how + // many of those land depends on timing (measured 4 unloaded, 7 on a constrained runner). The + // property is that the count does not scale with notifications - one migration, a handful of + // attempts - so the bound is deliberately loose rather than pinned to a number that will drift. + Assert.InRange(resubscribes, 1, 15); + + // Delivery is asserted by publishing *repeatedly*, because pub/sub is fire and forget: a message + // published while the subscription is still in flux is simply dropped. Losing messages during the + // tremor is expected; never delivering again is not, and one publish cannot tell those apart. + var delivered = await Poll.UntilAsync( + () => + { + if (Volatile.Read(ref received) > 0) return true; + conn.GetSubscriber().Publish(channel, "hello"); + return Volatile.Read(ref received) > 0; + }, + timeoutMilliseconds: 10_000, + pollMilliseconds: 250); + + log.WriteLine($"{mode}: delivered after migration = {delivered}"); + Assert.True(delivered, "the subscription should deliver again once things settle"); + } + } + + [Fact] + public async Task NodeThatLeavesTheClusterIsRetired() + { + // This is also the regression test for idleness counting *caller* work only. Before that, the + // reconfigure's own probes to the departed node piled into its backlog (~170 per pass, unbounded), so + // it looked busy because we were looking for it, and could never be retired. + + // The narrowed form of D5's "retire endpoints serving no slots". Serving nothing is *not* the + // condition: a node still listed in CLUSTER NODES is a live member that may be given slots again, and + // dropping its connection would be churn (go-redis does not). Having *left* the cluster is the + // condition, and the notification-driven refresh is what makes us notice. + EndPoint doomed = null!; + var (server, conn) = await ConnectAsync(log, configure: s => + { + GetHost(s.DefaultEndPoint, out var p); + doomed = s.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, p + 1)); + s.Migrate((RedisKey)"leaving", doomed); // owns something, so it is discovered at connect + }); + + using (server) + await using (conn) + { + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + GetHost(server.DefaultEndPoint, out var basePort); + var third = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, basePort + 2)); + + // hand its slot back and then remove it from the cluster outright + server.NotifyOnMigrate = true; + server.Migrate((RedisKey)"leaving", server.DefaultEndPoint); + Assert.True(server.RemoveNode(doomed), "the node should have been removed from the fake"); + + // Pruning wants several consecutive generations of absence. Driving those by *awaiting* topology + // passes rather than by sending notifications and sleeping: the notification path is covered by + // the tests above, and depending on it here would make this test wait out a jittered refresh per + // generation - which is what made it fail under a two-core runner. What is being tested is the + // retirement, so drive the generations deterministically. + // Retirement also requires the server to be *idle*, so a busy moment can defer it past a given + // pass - which is why this drives passes until it happens rather than asserting after a fixed + // count. Bounded, so a regression fails rather than hangs. + GC.KeepAlive(third); + var mux = (ConnectionMultiplexer)conn; + for (int i = 0; i < 20 && conn.GetEndPoints().Contains(doomed); i++) + { + await mux.ReconfigureAsync(first: false, reconfigureAll: true, log: null, blame: null, cause: $"test-generation-{i}"); + await Task.Delay(50); // retirement drains before removing, so give it a moment to complete + + } + + log.WriteLine($"endpoints: {string.Join(", ", conn.GetEndPoints().Select(x => x.ToString()))}"); + Assert.DoesNotContain(doomed, conn.GetEndPoints()); + } + } + + [Fact] + public async Task RefreshIsCoalescedRatherThanRepeated() + { + // A burst of notifications must not be a burst of topology reads. Note ReconfigureIfNeeded's own + // coalescing is *not* what achieves this: it declines only while a refresh is actually in flight, and + // the jitter spreads a burst out far enough that each pass finishes before the next starts. This test + // is what caught that, and why the coalescing happens before the delay rather than after. + var (server, conn) = await ConnectAsync(log); + using (server) + await using (conn) + { + GetHost(server.DefaultEndPoint, out var port); + var before = server.ClusterCommands; + + for (int i = 0; i < 10; i++) + { + server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [ + ($"127.0.0.1:{port}", $"127.0.0.1:{port + 1}", $"{i * 10}-{(i * 10) + 9}"), + ]); + } + + Assert.True(await UntilRefreshedAsync(server, before)); + await Task.Delay(2000); // let any stragglers land + + // a refresh reads both SLOTS and NODES, so allow a small multiple - the point is that ten + // notifications did not produce ten topology passes + var added = server.ClusterCommands - before; + log.WriteLine($"{added} cluster command(s) for 10 notifications"); + Assert.InRange(added, 1, 8); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs b/tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs new file mode 100644 index 000000000..d665d45e7 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/PeriodicTopologyRefreshTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Re-reading the topology on a timer, as a backstop for a change nothing reported. +/// +/// +/// The last of the three findings from the 37-hour field failure. The other two are closed: an endpoint that +/// only ever refuses now provokes a refresh after three consecutive connect failures, and a node that has left +/// the topology is retired rather than dialled forever. What neither covers is an endpoint that is +/// *reachable*, completes a handshake, and is no longer part of the deployment - it produces no failure, no +/// redirect and no announcement, so nothing asks the question. +/// +/// Non-parallel: these drive the multiplexer's own heartbeat and assert on what the server received, which a +/// neighbouring test's traffic would confuse. +/// +/// +[Collection(NonParallelCollection.Name)] +public class PeriodicTopologyRefreshTests(ITestOutputHelper log) +{ + /// Counts inbound CLUSTER commands, so a refresh is visible from the server's side. + private sealed class CountingServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _clusterCommands; + + public int ClusterCommands => Volatile.Read(ref _clusterCommands); + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + { + if (request.Count > 0 && string.Equals(request.GetString(0), "cluster", StringComparison.OrdinalIgnoreCase)) + { + Interlocked.Increment(ref _clusterCommands); + } + + return base.Execute(client, in request); + } + } + + private static async Task<(CountingServer Server, ConnectionMultiplexer Connection)> ConnectAsync( + ITestOutputHelper log, + int topologyRefreshSeconds) + { + var server = new CountingServer(log) { ServerType = ServerType.Cluster }; + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.TopologyRefreshSeconds = topologyRefreshSeconds; + + var conn = await ConnectionMultiplexer.ConnectAsync(config); + return (server, conn); + } + + [Fact] + public async Task TheTopologyIsReReadOnTheInterval() + { + // One second plus up to 30 of jitter, so this waits out the jitter rather than the interval: the + // interval is the part under test, and the jitter is what stops a fleet moving in step. + var (server, conn) = await ConnectAsync(log, topologyRefreshSeconds: 1); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + log.WriteLine($"cluster commands after connect: {before}"); + + Assert.True( + await Poll.UntilAsync(() => server.ClusterCommands > before, timeoutMilliseconds: 40_000, pollMilliseconds: 250), + "the topology should have been re-read without anything going wrong"); + + log.WriteLine($"cluster commands after the interval: {server.ClusterCommands}"); + } + } + + [Fact] + public async Task ZeroTurnsItOff() + { + // The escape hatch has to work, because this is the one refresh path whose cost is paid on a schedule + // rather than in response to something. Anybody who does not want it must be able to say so. + var (server, conn) = await ConnectAsync(log, topologyRefreshSeconds: 0); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + await Task.Delay(3000); + + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public async Task NothingIsReadBeforeTheFirstIntervalElapses() + { + // The interval runs from the first heartbeat, not from construction, so a multiplexer that is created, + // used and disposed inside it costs nothing. Asserted because the natural implementation - schedule at + // construction - makes short-lived multiplexers pay for a feature they never benefit from. + var (server, conn) = await ConnectAsync(log, topologyRefreshSeconds: 3600); + using (server) + await using (conn) + { + var before = server.ClusterCommands; + await Task.Delay(3000); + + log.WriteLine($"cluster commands: {before} -> {server.ClusterCommands}"); + Assert.Equal(before, server.ClusterCommands); + } + } + + [Fact] + public void TheDefaultIsLongAndSurvivesTheConfigurationString() + { + // 30 minutes: long enough that the fleet cost is negligible, short enough to bound how long a stale + // view can persist when nothing else notices. + Assert.Equal(1800, new ConfigurationOptions().TopologyRefreshSeconds); + + var parsed = ConfigurationOptions.Parse("localhost,topologyRefreshSeconds=120"); + Assert.Equal(120, parsed.TopologyRefreshSeconds); + Assert.Contains("topologyRefreshSeconds=120", parsed.ToString()); + + // ...and an explicit zero has to round-trip, or turning it off in a configuration string would look + // like it worked while silently reverting to the default + var disabled = ConfigurationOptions.Parse("localhost,topologyRefreshSeconds=0"); + Assert.Equal(0, disabled.TopologyRefreshSeconds); + Assert.Contains("topologyRefreshSeconds=0", disabled.ToString()); + + // Clone has to carry it too. ConfigTests.ExpectedFields is the "have you considered?" guard for this + // and it did its job - it caught the field being added without Clone knowing about it - but it checks + // that somebody looked, not that they got it right, so assert the behaviour as well. + Assert.Equal(0, disabled.Clone().TopologyRefreshSeconds); + Assert.Equal(120, parsed.Clone().TopologyRefreshSeconds); + } +} diff --git a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs index 748589e2b..f23efe5fe 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueStorageKindUnitTests.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Threading.Tasks; @@ -107,6 +107,6 @@ public void LiteralsResolveToExpectedStorageKinds() // only two storage flavors should ever appear; a third means something new crept in (e.g. a numeric // literal => Int64), which we want to notice Assert.Equal(2, byKind.Count); - Assert.Equal(22, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes + Assert.Equal(28, Count(RedisValue.StorageType.ByteArray)); // literals > 8 bytes } } diff --git a/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs b/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs new file mode 100644 index 000000000..38cea9ea8 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetirementUnderMaintenanceTests.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Configuration; +using StackExchange.Redis.Server; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests; + +/// +/// Retiring a node that refuses every connection while a sharded subscription was pinned to it, with a +/// maintenance window open - the shape the field ticket suggested could starve pruning indefinitely. +/// +/// +/// The worry, from the ticket: the customer's exception was on the Subscription bridge with +/// last: SSUBSCRIBE. Retirement requires the endpoint to be idle, idleness counts caller work, +/// and a sharded subscribe backlogged on a bridge that can never write it would be caller work - so the +/// topology could say the node is gone, pruning could want to retire it, and IsIdle() could veto it, +/// for longer while a maintenance window is open (relaxation raises the timeout that +/// CheckBacklogForTimeouts purges against). +/// +/// Measured, and the answer is reassuring: it does not happen, for a reason worth pinning down. Server +/// selection will not pick a disconnected node, so within a heartbeat the caller's subscribe is re-aimed at a +/// reachable sibling. What piles up on the refusing node is only our own traffic - autoconfigure +/// probes, keep-alives - which is exactly what HasCallerWork() was narrowed to exclude, so idleness is +/// true and the retirement proceeds. Both halves are asserted below, because the second is what makes the +/// first safe. +/// +/// +/// This also corrects the reading of the ticket: last: SSUBSCRIBE names the last command written +/// on that bridge, not a queued caller subscribe. The veto it implied needs a stronger precondition - no +/// reachable candidate for the slot at all - and at that point one endpoint's retirement is not the +/// interesting question. +/// +/// +[Collection(NonParallelCollection.Name)] +public class RetirementUnderMaintenanceTests(ITestOutputHelper log) +{ + [Fact] + public async Task ARefusingNodeAccumulatesOnlyOurOwnTrafficAndIsRetired() + { + using var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + var doomed = BlackHoleTunnel.GetRefusingEndPoint(); + server.AddEmptyNode(doomed); + + // the channel has to live on the doomed node, so pin it there by slot rather than by hope + var channel = RedisChannel.Sharded("retire-me"); + var asKey = (RedisKey)(byte[])channel!; + Assert.True(server.Migrate(asKey, doomed), "the fake should have moved the channel's slot"); + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = RedisProtocol.Resp3; + config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; + config.AbortOnConnectFail = false; + config.ReconnectRetryPolicy = new LinearRetry(500); + var tunnel = new BlackHoleTunnel(server.Tunnel); + config.Tunnel = tunnel; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var mux = (ConnectionMultiplexer)conn; + + Assert.True( + await Poll.UntilAsync(() => conn.GetEndPoints().Contains(doomed), timeoutMilliseconds: 10_000), + $"{doomed} was never discovered, so this test would prove nothing"); + + var subscriber = conn.GetSubscriber(); + await subscriber.SubscribeAsync(channel, (_, _) => { }); + Assert.True( + await Poll.UntilAsync(() => Equals(subscriber.SubscribedEndpoint(channel), doomed), timeoutMilliseconds: 5000), + $"the subscription should be pinned to {doomed}, but went to {subscriber.SubscribedEndpoint(channel)}"); + + var endpoint = mux.GetServerEndPoint(doomed, ServerProvenance.ClusterTopology); + + // A long window, opened while the node still answers - which is the only way it can be opened, and is + // the ordering the ticket had: the disruption is announced, and *then* the node goes away. + server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 60, shardIds: "[\"1\"]"); + Assert.True( + await Poll.UntilAsync(() => endpoint.IsMaintenanceRelaxed, timeoutMilliseconds: 5000), + "the notification should have opened a relaxed window on the doomed server"); + + // ...and now it goes away for good, refusing every connection. No notification of any of that, which + // is the point. + tunnel.BlackHole(doomed); + conn.GetServer(doomed).SimulateConnectionFailure(SimulatedFailureType.All); + + // Then *let the pressure build*, before the topology is allowed to catch up. This ordering is the + // whole test: while our slot map still says the channel lives on the doomed node, the resubscribe + // machinery keeps aiming SSUBSCRIBE at a bridge that can never write it, and those pile into the + // backlog as caller work. Move the slot first - as the first version of this test did - and the + // resubscribe goes to the survivor instead, nothing accumulates, and the test passes having exercised + // nothing. The ticket's client was in exactly this state: dead endpoint, stale map, pending SSUBSCRIBE. + // Let it sit *before* the topology is allowed to catch up. This ordering is the whole test: while our + // slot map still says the channel lives on the doomed node, anything aimed at that slot is aimed at a + // bridge that can never write it. Move the slot first and the question never arises. + var survivor = mux.GetServerEndPoint(server.DefaultEndPoint, ServerProvenance.ClusterTopology); + await Task.Delay(3000); + log.WriteLine( + $"under pressure - doomed: callerWork={endpoint.HasCallerWork()} outstanding={endpoint.GetOutstandingCount()} " + + $"relaxed={endpoint.IsMaintenanceRelaxed} idle={endpoint.IsIdle()}; " + + $"subscribed to {subscriber.SubscribedEndpoint(channel)?.ToString() ?? "(nowhere yet)"}"); + + // The refusing node *is* accumulating work - if it were not, the rest of this proves nothing, because + // the distinction being tested would be vacuous... + Assert.True( + endpoint.GetOutstandingCount() > 0, + "the refusing node should be accumulating our own probe traffic; with nothing outstanding this test " + + "is not exercising the distinction that lets retirement proceed"); + + // ...and none of it is a caller's, which is what keeps idleness true. Selection will not pick a + // disconnected node, so the caller's subscribe was re-aimed at the reachable sibling within a + // heartbeat rather than queueing here for the relaxed timeout to eventually purge. + Assert.False( + endpoint.HasCallerWork(), + "a caller's work should not be queued on a node that cannot be written to while a reachable " + + "candidate for the slot exists"); + Assert.NotEqual(doomed, subscriber.SubscribedEndpoint(channel)); + GC.KeepAlive(survivor); + + // only now does the cluster admit it has gone + Assert.True(server.Migrate(asKey, server.DefaultEndPoint)); + Assert.True(server.RemoveNode(doomed), "the node should have been removed from the fake"); + + // Generations are driven rather than waited for, as in the quiet case: what is under test is the + // retirement, not the refresh trigger. Bounded so a regression fails rather than hangs. + for (int i = 0; i < 40 && conn.GetEndPoints().Contains(doomed); i++) + { + await mux.ReconfigureAsync(first: false, reconfigureAll: true, log: null, blame: null, cause: $"test-generation-{i}"); + if (i % 8 == 0) + { + log.WriteLine( + $"generation {i}: idle={endpoint.IsIdle()} callerWork={endpoint.HasCallerWork()} " + + $"outstanding={endpoint.GetOutstandingCount()} relaxed={endpoint.IsMaintenanceRelaxed} " + + $"subs={endpoint.GetCounters().Subscription.Subscriptions}"); + } + + await Task.Delay(100); + } + + log.WriteLine($"endpoints: {string.Join(", ", conn.GetEndPoints().Select(x => x.ToString()))}"); + log.WriteLine( + $"final: idle={endpoint.IsIdle()} callerWork={endpoint.HasCallerWork()} " + + $"outstanding={endpoint.GetOutstandingCount()} relaxed={endpoint.IsMaintenanceRelaxed}"); + + Assert.DoesNotContain(doomed, conn.GetEndPoints()); + + // and the caller's subscription is not collateral damage: it belongs to the surviving node now + Assert.True( + await Poll.UntilAsync(() => Equals(subscriber.SubscribedEndpoint(channel), server.DefaultEndPoint), timeoutMilliseconds: 10_000), + $"the subscription should have moved to the surviving node, but is on {subscriber.SubscribedEndpoint(channel)}"); + } +} diff --git a/tests/StackExchange.Redis.Tests/TestBase.cs b/tests/StackExchange.Redis.Tests/TestBase.cs index c59039478..505f2bb4e 100644 --- a/tests/StackExchange.Redis.Tests/TestBase.cs +++ b/tests/StackExchange.Redis.Tests/TestBase.cs @@ -454,7 +454,7 @@ public static ConnectionMultiplexer CreateDefault( log ??= localLog = new StringWriter(); try { - var config = ConfigurationOptions.Parse(configuration); + var config = TestConfig.ApplyMaintenanceDefault(ConfigurationOptions.Parse(configuration)); if (disabledCommands != null && disabledCommands.Length != 0) { config.CommandMap = CommandMap.Create([.. disabledCommands], false); diff --git a/toys/MaintenanceSoak/Invariants.cs b/toys/MaintenanceSoak/Invariants.cs new file mode 100644 index 000000000..6888aa6f4 --- /dev/null +++ b/toys/MaintenanceSoak/Invariants.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace StackExchange.Redis.MaintenanceSoak; + +/// +/// The things that must stay true however many cycles run. +/// +/// +/// These are the failure modes a unit test structurally cannot reach, because each needs *repetition* to +/// appear: state that accumulates, a flag that is set and never cleared, a window that is extended once too +/// often. Each violation is recorded with the cycle it happened on rather than throwing, so one run reports +/// everything it found instead of the first thing. +/// +internal sealed class Invariants +{ + private readonly List _violations = []; + + public IReadOnlyList Violations => _violations; + + public void Check(bool condition, int cycle, string what) + { + if (!condition) _violations.Add($"cycle {cycle}: {what}"); + } + + /// + /// Whether the same violation has already been recorded, so a systemic failure reports once per kind + /// rather than once per cycle. + /// + public bool AlreadySeen(string what) => _violations.Exists(v => v.EndsWith(what, StringComparison.Ordinal)); + + public void Record(int cycle, string what) + { + if (!AlreadySeen(what)) _violations.Add($"cycle {cycle}: {what}"); + } +} + +/// +/// A memory sample, taken with a forced collection so the number means something. +/// +internal readonly record struct MemorySample(int Cycle, long Bytes) +{ + public static MemorySample Take(int cycle) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + return new MemorySample(cycle, GC.GetTotalMemory(forceFullCollection: true)); + } +} diff --git a/toys/MaintenanceSoak/MaintenanceSoak.csproj b/toys/MaintenanceSoak/MaintenanceSoak.csproj new file mode 100644 index 000000000..0591dff3c --- /dev/null +++ b/toys/MaintenanceSoak/MaintenanceSoak.csproj @@ -0,0 +1,21 @@ + + + + Exe + net10.0 + enable + StackExchange.Redis.MaintenanceSoak + + true + + + $(NoWarn);SER010;StringToRedisValue + + + + + + + + diff --git a/toys/MaintenanceSoak/Program.cs b/toys/MaintenanceSoak/Program.cs new file mode 100644 index 000000000..a7d16999c --- /dev/null +++ b/toys/MaintenanceSoak/Program.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; +using StackExchange.Redis.MaintenanceSoak; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Server; +using static StackExchange.Redis.Server.RedisServer; + +// Soak for the maintenance-notification machinery: continuous traffic while notifications are injected on a +// loop, watching for the things that only appear with repetition. +// +// Why this exists, when the feature already has unit tests and a live scenario tier: stage 2 onwards introduced +// real state with lifetimes - relaxed windows that extend, a post-event tail, an eight-slot dedup ring, a +// handoff in-flight flag, a handoff target with an expiry - and its failure modes are "a window never closes", +// "a flag is never cleared", "something accumulates". None of those show up once; all of them show up on the +// thousandth cycle. That is the gap this fills, and it is what discharges NF.1. +// +// Usage: MaintenanceSoak [cycles] [--workers N] [--port N] + +int cycles = 500, workers = 8, port = 0; +for (int i = 0; i < args.Length; i++) +{ + if (args[i] == "--workers" && i + 1 < args.Length) workers = int.Parse(args[++i]); + else if (args[i] == "--port" && i + 1 < args.Length) port = int.Parse(args[++i]); + else if (int.TryParse(args[i], out var parsed)) cycles = parsed; +} + +Console.WriteLine($"soak: {cycles} cycles, {workers} workers"); + +var invariants = new Invariants(); +await using var host = new SoakServer(new MemoryCacheRedisServer(), port); +Console.WriteLine($"server listening on {host.EndPoint}"); + +var options = new ConfigurationOptions +{ + EndPoints = { host.EndPoint }, + Protocol = RedisProtocol.Resp3, + MaintenanceNotifications = MaintenanceNotificationMode.Enabled, + AbortOnConnectFail = false, + ConnectTimeout = 5_000, + SyncTimeout = 5_000, + AllowAdmin = true, + + // Short windows on purpose. The defaults are a 10s floor with a 20s tail, which is right for a real + // deployment and useless here: the soak injects continuously, so it would spend its whole life inside one + // window and never observe one *closing*. One second exercises the same code with a tractable clock. + MaintenanceRelaxedTimeout = TimeSpan.FromSeconds(1), +}; + +await using var muxer = await ConnectionMultiplexer.ConnectAsync(options); +var server = ((IInternalConnectionMultiplexer)muxer).GetServerEndPoint(host.EndPoint); +Console.WriteLine($"connected; maintenance notifications active: {server.MaintenanceNotificationsActive}"); +if (!server.MaintenanceNotificationsActive) +{ + Console.Error.WriteLine("FAIL: the opt-in was not accepted, so this run would prove nothing"); + return 2; +} + +// ---- continuous traffic ------------------------------------------------------------------------------------- +using var running = new CancellationTokenSource(); +long commands = 0, failures = 0; +var traffic = Enumerable.Range(0, workers).Select(worker => Task.Run(async () => +{ + var db = muxer.GetDatabase(); + var key = (RedisKey)$"soak-{worker}"; + while (!running.IsCancellationRequested) + { + try + { + await db.StringSetAsync(key, Guid.NewGuid().ToString("n")); + await db.StringGetAsync(key); + Interlocked.Add(ref commands, 2); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // Expected: the soak severs connections on purpose. Counted, not fatal - what matters is that + // traffic recovers, which the throughput check below covers. + Interlocked.Increment(ref failures); + await Task.Delay(10); + } + } +})).ToArray(); + +// ---- notification counting --------------------------------------------------------------------------------- +long events = 0; +muxer.ServerMaintenanceEvent += (_, e) => +{ + if (e is PushMaintenanceEvent) Interlocked.Increment(ref events); +}; + +// ---- the loop ---------------------------------------------------------------------------------------------- +var baseline = MemorySample.Take(0); +var samples = new List { baseline }; +var watch = Stopwatch.StartNew(); +long expectedEvents = 0; +int seq = 0; + +for (int cycle = 1; cycle <= cycles; cycle++) +{ + // Rotate through the shapes, so no single kind dominates and the pairs interleave. + // + // Note every Send returns how many opted-in clients it actually reached, and that is what counts as "sent" + // - not the number of calls. A MOVING handoff deliberately replaces the connection, so a notification + // issued during that instant has nobody to go to and returns zero. Counting calls instead reported a lost + // event on a run that had lost nothing. + switch (cycle % 5) + { + case 0: + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.Migrating, timeSeconds: 1, shardIds: "[\"1\"]", sequenceId: seq++); + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.Migrated, timeSeconds: null, shardIds: "[\"1\"]", sequenceId: seq++); + break; + case 1: + expectedEvents += host.Server.SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, "0-100", sequenceId: seq++); + expectedEvents += host.Server.SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [($"{host.EndPoint.Address}:{host.EndPoint.Port}", $"{host.EndPoint.Address}:{host.EndPoint.Port}", "0-100")], sequenceId: seq++); + break; + case 2: + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.FailingOver, timeSeconds: 1, shardIds: "[\"2\"]", sequenceId: seq++); + expectedEvents += host.Server.SendShardNotification(null, MaintenanceNotificationKind.FailedOver, timeSeconds: null, shardIds: "[\"2\"]", sequenceId: seq++); + break; + case 3: + // MOVING with no successor: the DNS/half-window path, and it replaces connections + expectedEvents += host.Server.SendMoving(null, timeSeconds: 1, newEndpoint: null, sequenceId: seq++); + break; + default: + // MOVING naming somewhere to go: the handoff-target path + expectedEvents += host.Server.SendMoving(null, timeSeconds: 1, newEndpoint: host.EndPoint, sequenceId: seq++); + break; + } + + await Task.Delay(25); + + if (cycle % 50 == 0) + { + // Mid-storm the window *should* be open: notifications are arriving faster than it can expire, so an + // endpoint that is not relaxed here means they are being received and ignored. + invariants.Check(server.IsMaintenanceRelaxed, cycle, "timeouts were not relaxed during continuous notifications"); + + // ...and then it must close once they stop. This is the check that needs the quiet: asking whether a + // window has closed while still injecting only measures the injection rate, which is how the first + // version of this reported a violation on every checkpoint of a perfectly healthy run. + var settled = await SettlesAsync(() => !server.IsMaintenanceRelaxed, TimeSpan.FromSeconds(10)); + invariants.Check(settled, cycle, "the relaxed window never closed after notifications stopped (a stuck timeout)"); + + // A handoff must never be left in flight either: the flag is what stops a second one starting, so a + // leak means no future MOVING is ever acted on again - silently. + invariants.Check(server.HandoffTarget is null, cycle, "a handoff target outlived its window (would pin this endpoint to one address)"); + + // the dedup ring is fixed-size, so what is worth checking is that collapsing still *works* after + // thousands of events rather than silently dropping everything + var seen = Interlocked.Read(ref events); + invariants.Check(seen > 0, cycle, "no notifications were raised at all"); + + samples.Add(MemorySample.Take(cycle)); + var clients = host.Server.ClientCount; + Console.WriteLine( + $" cycle {cycle,5}: events={seen,6} commands={Interlocked.Read(ref commands),8} " + + $"failures={Interlocked.Read(ref failures),4} clients={clients,2} handoffs={server.HandoffRecycles,4} " + + $"mem={samples[^1].Bytes / 1024,7}KB"); + + // A recycle closes the old connection and opens one; the fake should not be accumulating them. + invariants.Check(clients <= 8, cycle, $"the server is holding {clients} clients, which suggests connections are not being released"); + } +} + +running.Cancel(); +await Task.WhenAll(traffic); +watch.Stop(); + +// ---- report ------------------------------------------------------------------------------------------------- +var first = samples.Count > 1 ? samples[1] : baseline; // after warmup +var last = samples[^1]; +var growth = first.Bytes == 0 ? 0 : (last.Bytes - first.Bytes) * 100.0 / first.Bytes; + +Console.WriteLine(); +Console.WriteLine($"cycles: {cycles} in {watch.Elapsed.TotalSeconds:0.0}s"); +Console.WriteLine($"notifications: {Interlocked.Read(ref events)} raised, {expectedEvents} delivered"); +Console.WriteLine($"commands: {Interlocked.Read(ref commands)} ok, {Interlocked.Read(ref failures)} failed"); +Console.WriteLine($"memory: {first.Bytes / 1024}KB -> {last.Bytes / 1024}KB ({growth:+0.0;-0.0;0}%)"); +Console.WriteLine($"clients: {host.Server.ClientCount} now, {host.Server.TotalClientCount} over the run"); + +// Far fewer than the number of MOVINGs sent, and that is correct: a MOVING arriving while a handoff is already +// in flight is ignored, since starting a second poll against the same endpoint achieves nothing. Reported so +// the number is visible rather than assumed - a zero here would mean the handoff path never ran at all. +Console.WriteLine($"handoffs: {server.HandoffRecycles} connection replacements"); + +// Growth is reported always and failed only when egregious: a soak that cries wolf on GC noise gets ignored, +// which is worse than not having one. +if (growth > 50 && last.Bytes - first.Bytes > 32 * 1024 * 1024) +{ + invariants.Record(cycles, $"managed memory grew {growth:0}% ({(last.Bytes - first.Bytes) / 1024 / 1024}MB) after warmup"); +} + +// Every notification the server delivered should be raised exactly once - with one bounded exception. A +// handoff replaces the connection, and a frame already queued for the socket it replaces is lost: the server +// counted a write, but nobody was left to read it. So loss is tolerated up to the number of connection +// replacements, and anything beyond that is a real leak in the receive path. Over-raising is never tolerated: +// it would mean the collapse has stopped collapsing. +var raised = Interlocked.Read(ref events); +var lost = expectedEvents - raised; +var handoffs = server.HandoffRecycles; +Console.WriteLine($"unraised: {lost} (tolerance {handoffs}, one per connection replacement)"); + +if (raised > expectedEvents) +{ + invariants.Record(cycles, $"{raised} notifications raised for only {expectedEvents} delivered - the collapse is not collapsing"); +} +else if (lost > handoffs) +{ + invariants.Record(cycles, $"{lost} delivered notifications were never raised, with only {handoffs} connection replacements to explain them"); +} + +if (server.HandoffRecycles == 0) +{ + invariants.Record(cycles, "no handoff ever ran, so this run says nothing about the handoff path"); +} + +if (invariants.Violations.Count == 0) +{ + Console.WriteLine(); + Console.WriteLine("PASS: no invariant violated"); + return 0; +} + +Console.WriteLine(); +Console.Error.WriteLine($"FAIL: {invariants.Violations.Count} invariant violation(s)"); +foreach (var violation in invariants.Violations) Console.Error.WriteLine($" {violation}"); +return 1; + +static async Task SettlesAsync(Func condition, TimeSpan timeout) +{ + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (condition()) return true; + await Task.Delay(100); + } + + return condition(); +} diff --git a/toys/MaintenanceSoak/SoakServer.cs b/toys/MaintenanceSoak/SoakServer.cs new file mode 100644 index 000000000..54668de2e --- /dev/null +++ b/toys/MaintenanceSoak/SoakServer.cs @@ -0,0 +1,110 @@ +using System; +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Server; + +namespace StackExchange.Redis.MaintenanceSoak; + +/// +/// Hosts the in-process RESP server on a real TCP socket. +/// +/// +/// A real socket rather than the tunnel the test suite uses, deliberately: a soak is looking for what leaks +/// over thousands of cycles - sockets, pipes, buffers, event handlers - and the tunnel bypasses exactly the +/// machinery most likely to leak. It also means the connection has a genuine remote address, so the handoff's +/// endpoint-type derivation runs for real instead of resolving to "no address to classify". +/// +internal sealed class SoakServer : IAsyncDisposable +{ + private readonly Socket _listener; + private readonly CancellationTokenSource _shutdown = new(); + private readonly Task _accepting; + + public SoakServer(MemoryCacheRedisServer server, int port) + { + Server = server; + _listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + _listener.Bind(new IPEndPoint(IPAddress.Loopback, port)); + _listener.Listen(64); + EndPoint = (IPEndPoint)_listener.LocalEndPoint!; + _accepting = AcceptLoopAsync(); + } + + public MemoryCacheRedisServer Server { get; } + + public IPEndPoint EndPoint { get; } + + private async Task AcceptLoopAsync() + { + while (!_shutdown.IsCancellationRequested) + { + Socket socket; + try + { + socket = await _listener.AcceptAsync(_shutdown.Token); + } + catch (Exception) when (_shutdown.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + Console.WriteLine($"accept failed: {ex.Message}"); + continue; + } + + _ = ServeAsync(socket); + } + } + + private async Task ServeAsync(Socket socket) + { + try + { + socket.NoDelay = true; + using var stream = new NetworkStream(socket, ownsSocket: true); + + // one pipe each way, which is what RunClientAsync expects + var input = PipeReader.Create(stream); + var output = PipeWriter.Create(stream); + await Server.RunClientAsync(new DuplexPipe(input, output)); + } + catch (Exception ex) when (ex is System.IO.IOException or ObjectDisposedException or SocketException) + { + // an ordinary disconnect; the soak causes plenty of them on purpose + } + catch (Exception ex) + { + Console.WriteLine($"client faulted: {ex.GetType().Name}: {ex.Message}"); + } + } + + public async ValueTask DisposeAsync() + { + _shutdown.Cancel(); + try + { + _listener.Dispose(); + } + catch { } + + try + { + await _accepting; + } + catch { } + + Server.Dispose(); + _shutdown.Dispose(); + } + + private sealed class DuplexPipe(PipeReader input, PipeWriter output) : IDuplexPipe + { + public PipeReader Input => input; + public PipeWriter Output => output; + } +} diff --git a/toys/MaintenanceWatch/MaintenanceWatch.csproj b/toys/MaintenanceWatch/MaintenanceWatch.csproj new file mode 100644 index 000000000..b400a6ced --- /dev/null +++ b/toys/MaintenanceWatch/MaintenanceWatch.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + + $(NoWarn);SER010 + + + + + + + diff --git a/toys/MaintenanceWatch/Program.cs b/toys/MaintenanceWatch/Program.cs new file mode 100644 index 000000000..f75aacdec --- /dev/null +++ b/toys/MaintenanceWatch/Program.cs @@ -0,0 +1,115 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; +using StackExchange.Redis.Maintenance; + +// Watches a real deployment for server-native maintenance notifications and prints what we made of them. +// +// The point of this tool is *evidence*: that we asked for notifications, that they arrive, and that each one +// was understood - so every event prints both our parsed view and the raw payload it came from, and any +// disagreement between the two is visible rather than inferred. +// +// dotnet run --project toys/MaintenanceWatch -- "my-endpoint:6379,user=...,password=..." +// +// Notes: +// - RESP3 is required; this forces it rather than relying on the endpoint's provider defaults. +// - The opt-in mode is forced to Auto, so an endpoint that does not support notifications still connects. +// - Nothing here reacts to a notification; observing is the whole job. +if (args.Length == 0) +{ + Console.Error.WriteLine("usage: MaintenanceWatch [--enabled]"); + return 1; +} + +var config = ConfigurationOptions.Parse(args[0]); +config.Protocol = RedisProtocol.Resp3; +config.MaintenanceNotifications = args.Contains("--enabled") + ? MaintenanceNotificationMode.Enabled // refuse to run if the server will not deliver + : MaintenanceNotificationMode.Auto; +config.AbortOnConnectFail = false; + +// note ToString() includes the password by default; the library's own logging masks it (see +// LoggerExtensions), but this line is ours, so it has to ask +Console.WriteLine($"connecting: {config.ToString(includePassword: false)}"); +Console.WriteLine($"defaults provider: {config.Defaults}"); +Console.WriteLine($"maintNotifications={config.MaintenanceNotifications}, relaxed={config.MaintenanceRelaxedTimeout.TotalSeconds}s," + + $" windowMax={config.MaintenanceRelaxedWindowMax.TotalSeconds}s, postEvent={config.MaintenancePostEventRelaxedDuration.TotalSeconds}s"); + +// the connect log carries the negotiation half: whether the opt-in was sent, and how the server answered +await using var conn = await ConnectionMultiplexer.ConnectAsync(config, Console.Out); + +var count = 0; +conn.ServerMaintenanceEvent += (_, e) => +{ + var n = Interlocked.Increment(ref count); + Console.WriteLine(); + Console.WriteLine($"=== maintenance event #{n} at {DateTime.UtcNow:HH:mm:ss.fff}Z ==="); + Console.WriteLine($" raw: {e.RawMessage}"); + + if (e is not PushMaintenanceEvent push) + { + // e.g. the Azure pub/sub channel; not what this tool is for, but worth seeing rather than hiding + Console.WriteLine($" (not a push notification: {e.GetType().Name})"); + return; + } + + Console.WriteLine($" type: {push.NotificationType} seq: {push.SequenceId} from: {push.EndPoint}"); + Console.WriteLine($" time: {(push.Time is { } t ? $"{t.TotalSeconds}s" : "(none)")}" + + $" startsAt: {(push.StartTimeUtc is { } at ? $"{at:HH:mm:ss}Z" : "(n/a)")}"); + + if (push.NotificationType == MaintenanceNotificationType.Moving) + { + // null is a documented outcome, not a parse failure: reconnect to what you already have + Console.WriteLine($" moving to: {push.NewEndPoint?.ToString() ?? "(no address given)"}"); + } + + if (push.Payload is { } payload) + { + Console.WriteLine($" payload: {payload}"); + } + + foreach (var migration in push.SlotMigrations) + { + var slots = migration.Slots.Count == 0 + ? $"(unparsed: '{migration.RawSlots}')" + : string.Join(",", migration.Slots.Select(x => x.From == x.To ? $"{x.From}" : $"{x.From}-{x.To}")); + Console.WriteLine($" slots {slots}: {migration.Source?.ToString() ?? "?"} -> {migration.Target?.ToString() ?? "?"}"); + } +}; + +// a fault during an announced disruption says so, which is the other half of "we understood it" +conn.ErrorMessage += (_, e) => Console.WriteLine($"[error] {e.EndPoint}: {e.Message}"); +conn.ConnectionFailed += (_, e) => Console.WriteLine($"[failed] {e.EndPoint}: {e.FailureType} {e.Exception?.Message}"); +conn.ConnectionRestored += (_, e) => Console.WriteLine($"[restored] {e.EndPoint}"); + +Console.WriteLine(); +Console.WriteLine("watching; a little traffic keeps the connection interesting. Ctrl+C to stop."); + +var db = conn.GetDatabase(); +var key = $"maintenance-watch:{Guid.NewGuid():N}"; +while (true) +{ + try + { + await db.StringSetAsync(key, (RedisValue)DateTime.UtcNow.Ticks); + _ = await db.StringGetAsync(key); + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + // note RedisTimeoutException derives from TimeoutException, *not* RedisException - so a + // `catch (RedisException)` would silently miss exactly the case this tool exists to show + // MaintenanceType is the payoff here: "timeout" versus "timeout during an announced failover" + var maintenance = ex switch + { + RedisTimeoutException timeout => timeout.MaintenanceType, + RedisConnectionException connection => connection.MaintenanceType, + _ => MaintenanceNotificationType.None, + }; + Console.WriteLine($"[command] {ex.GetType().Name}: {ex.Message}" + + (maintenance == MaintenanceNotificationType.None ? "" : $" <- during {maintenance}")); + } + + await Task.Delay(1000); +} diff --git a/toys/StackExchange.Redis.Server/RedisClient.Output.cs b/toys/StackExchange.Redis.Server/RedisClient.Output.cs index 0e9396a5a..b0a2b21d6 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.Output.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.Output.cs @@ -27,6 +27,30 @@ private readonly struct VersionedResponse(TypedRedisValue value, RedisProtocol p private readonly Channel _replies = Channel.CreateUnbounded(s_replyChannelOptions); + private TypedRedisValue _deferred = TypedRedisValue.Nil; + + /// + /// Queues a frame to be sent *after* the reply to the command currently being executed. + /// + /// + /// Needed because the read loop enqueues a command's reply once Execute has returned, so a handler + /// that calls directly puts its frame *before* its own reply. A real server does + /// the opposite - the retained maintenance notification arrives immediately after the +OK for the + /// opt-in, in the same TCP segment - and the ordering is the whole point of modelling this. + /// + public void AddOutboundAfterReply(in TypedRedisValue message) + { + FlushDeferredOutbound(); // one slot is enough for any real case; do not lose an earlier frame + _deferred = message; + } + + internal void FlushDeferredOutbound() + { + var pending = _deferred; + _deferred = TypedRedisValue.Nil; + if (!pending.IsNil) AddOutbound(pending); + } + public void AddOutbound(in TypedRedisValue message) { if (message.IsNil) @@ -139,6 +163,9 @@ public async Task WriteOutputAsync(PipeWriter writer) } catch (Exception ex) { + // this used to vanish into the pipe: a serialization bug in a fake server's *outbound* path + // presents as "the client never received it", with a reconnect covering the tracks. Log it + Node?.Server?.Log($"[{this}] write loop faulted: {ex.Message}"); await writer.CompleteAsync(ex); } } diff --git a/toys/StackExchange.Redis.Server/RedisClient.cs b/toys/StackExchange.Redis.Server/RedisClient.cs index d3e2c1475..bed7b00f0 100644 --- a/toys/StackExchange.Redis.Server/RedisClient.cs +++ b/toys/StackExchange.Redis.Server/RedisClient.cs @@ -99,6 +99,22 @@ public bool TryReadRequest(ReadOnlySequence data, out long consumed) } public RedisServer.Node Node => node; + + /// + /// Whether this connection has opted in to maintenance notifications, and with which endpoint type - + /// per connection, because that is how the real opt-in is scoped, so a test can assert that every + /// connection opted in rather than merely that one did. + /// + public bool MaintenanceNotifications { get; internal set; } + + /// The moving-endpoint-type this connection asked for, or null for server defaults. + public string MovingEndpointType { get; internal set; } + + /// + /// How many times this connection has sent the opt-in. Re-arming on reconnect is a requirement, and a + /// count is what distinguishes "opted in once" from "opted in again after reconnecting". + /// + public int MaintenanceNotificationOptInCount { get; internal set; } public int SkipReplies { get; set; } public void SkipAllReplies() => SkipReplies = -1; internal bool ShouldSkipResponse() @@ -141,7 +157,12 @@ public void Dispose() } private int _activeSlot = ServerSelectionStrategy.NoSlot; - internal void ResetAfterRequest() => _activeSlot = ServerSelectionStrategy.NoSlot; + internal void ResetAfterRequest() + { + _activeSlot = ServerSelectionStrategy.NoSlot; + FlushDeferredOutbound(); // after the reply for this command, which the read loop has now queued + } + public virtual void OnKey(in RedisKey key, KeyFlags flags) { if ((flags & KeyFlags.NoSlotCheck) == 0 & node.CheckCrossSlot) diff --git a/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs new file mode 100644 index 000000000..7f3878309 --- /dev/null +++ b/toys/StackExchange.Redis.Server/RedisServer.Maintenance.cs @@ -0,0 +1,440 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite; +using RESPite.Messages; + +namespace StackExchange.Redis.Server +{ + /// + /// Sending maintenance notifications. Deliberately on the server itself rather than on a bespoke test + /// subclass: the opt-in is a normal command any test may use, and injecting a notification is a normal + /// thing any test may want to do. A server that never sends one is the ordinary case - that is what every + /// OSS build does - so the interesting behaviour is on the client side either way. + /// + public partial class RedisServer + { + /// + /// The notification types defined by the maintenance-notification contract. SMOVING and + /// SFAILING_OVER were proposed upstream but never landed, and are deliberately absent. + /// + public enum MaintenanceNotificationKind + { + /// This endpoint is being replaced; the payload names its successor. + Moving, + + /// A shard is migrating away from this node. + Migrating, + + /// The migration has completed. + Migrated, + + /// This node is failing over. + FailingOver, + + /// The failover has completed. + FailedOver, + + /// Slots are migrating (OSS cluster family). + SlotMigrating, + + /// Slots have migrated (OSS cluster family). + SlotMigrated, + } + + private static string GetName(MaintenanceNotificationKind kind) => kind switch + { + MaintenanceNotificationKind.Moving => "MOVING", + MaintenanceNotificationKind.Migrating => "MIGRATING", + MaintenanceNotificationKind.Migrated => "MIGRATED", + MaintenanceNotificationKind.FailingOver => "FAILING_OVER", + MaintenanceNotificationKind.FailedOver => "FAILED_OVER", + MaintenanceNotificationKind.SlotMigrating => "SMIGRATING", + MaintenanceNotificationKind.SlotMigrated => "SMIGRATED", + _ => throw new ArgumentOutOfRangeException(nameof(kind)), + }; + + /// + /// An artificial delay applied before each reply, to make command timeouts happen on demand. + /// + /// + /// This is how "does relaxation actually save a command?" gets tested. The obvious alternative - the + /// fault injector's network_latency action against a real cluster - turns out to be the wrong + /// tool: it applies netem to a whole *node's* interface, so it delays the cluster's internal traffic + /// and its DNS as well as the client's, and 200ms across two of three nodes was enough to take a + /// working deployment offline. Its duration_seconds did not revert either. A per-connection + /// delay here is precise, instant, and cannot break anything. + /// + /// Applies to *every* reply on the connection, including a handshake, so set it after connecting + /// unless a slow handshake is what is being tested. + /// + /// + public TimeSpan ResponseDelay { get; set; } + + /// + /// + /// Not an async method: the base signature takes the request by in, which async forbids. + /// + protected override ValueTask ClientPauseAsync(RedisClient client, in RedisRequest request) + { + var delay = ResponseDelay; + return delay > TimeSpan.Zero ? new ValueTask(Task.Delay(delay)) : default; + } + + private int _maintenanceSequence; + private int _maintenanceOptIns; + private (MaintenanceNotificationKind Kind, long Sequence, string ShardIds)? _retainedCompletion; + + /// + /// Whether the server keeps the most recent shard-scoped *completion* and replays it to each + /// connection that opts in, as Redis Enterprise does. + /// + /// + /// Observed on RS 8.0.22 (2026-08-28), and the boundary is sharp: MIGRATED and + /// FAILED_OVER are retained; MIGRATING, FAILING_OVER, MOVING, + /// SMIGRATING and SMIGRATED are not. The characterisation that fits all seven is the + /// completion of a *shard-scoped* event - the two that carry an affected-shards list - so + /// SMIGRATED (a slot-range triple) and MOVING (an endpoint) are excluded even though + /// SMIGRATED is also a completion. + /// + /// The consequence is a design property worth stating: the catch-up channel can only ever say "a + /// disruption ended", never "one is starting". Nothing that demands action is replayed, so a + /// reconnecting client cannot be told to move by a stale frame. + /// + /// + /// Retained "most recent, replaced not accumulated" - one frame, never a queue - so a connection sees + /// at most one of these however many events went past. + /// + /// + public bool RetainCompletions { get; set; } = true; + + /// + /// Whether this notification is one of the two the server retains for replay. + /// + private static bool IsRetainedCompletion(MaintenanceNotificationKind kind) + => kind is MaintenanceNotificationKind.Migrated or MaintenanceNotificationKind.FailedOver; + + /// + /// Replays the retained completion, if any, to a client that has just opted in. + /// + /// + /// Deliberately *after* the +OK (see ), and with + /// the original sequence id rather than a fresh one: the id identifies the event, and a client using it + /// for dedup has to be able to recognize a completion it has already seen. + /// + private void ReplayRetainedCompletion(RedisClient client) + { + if (!RetainCompletions || _retainedCompletion is not { } retained) return; + + client.AddOutboundAfterReply(BuildShardNotification(retained.Kind, null, retained.Sequence, null, retained.ShardIds)); + Log($"[{client}] replayed retained {GetName(retained.Kind)} (seq {retained.Sequence})"); + } + + /// + /// How many times a client has opted in, across the lifetime of the server. Per-client state goes away + /// with the client, so this is what a reconnect test can measure. + /// + public int TotalMaintenanceOptIns => Volatile.Read(ref _maintenanceOptIns); + + internal void OnMaintenanceOptIn() => Interlocked.Increment(ref _maintenanceOptIns); + + /// + /// Whether announces itself the way a real server does, rather + /// than only moving the slot in this model. + /// + /// + /// Off by default, because plenty of tests use Migrate purely to arrange a topology and would + /// not expect a notification to arrive mid-arrangement. Turn it on to exercise the *sequence* a client + /// really sees - which is the only way to test how the notification-driven path and the unsolicited + /// SUNSUBSCRIBE path interact, since both fire for the same migration. + /// + public bool NotifyOnMigrate { get; set; } + + /// + /// Announces a slot migration the way a real server does: the shard notifications either side of it, + /// and an unsolicited sunsubscribe to any client subscribed to a sharded channel that has just + /// moved away. + /// + /// + /// Note the two signals are independent. The notifications only reach clients that opted in, whereas + /// the unsubscribe is ordinary cluster behaviour and reaches every subscriber - so a client can + /// legitimately see one, the other, or both, and the order is a server implementation detail. That is + /// exactly the interaction worth being able to reproduce here. + /// + private void AnnounceMigration(int hashSlot, Node from, Node to) + { + var slots = hashSlot.ToString(System.Globalization.CultureInfo.InvariantCulture); + SendSlotNotification(null, MaintenanceNotificationKind.SlotMigrating, slots); + + var dropped = ForAllClients(hashSlot, static (client, slot) => client.UnsubscribeMigratedSlot(slot)); + if (dropped != 0) Log($"unsubscribed {dropped} sharded subscription(s) for migrated slot {hashSlot}"); + + SendSlotMigrations(null, MaintenanceNotificationKind.SlotMigrated, + [($"{from.Host}:{from.Port}", $"{to.Host}:{to.Port}", slots)]); + } + + /// + /// The sequence id given to the next notification, unless one is supplied explicitly. The contract does + /// not define these, so a client's use of them is its own invention - which is worth being able to + /// exercise, including by repeating one. + /// + public int NextMaintenanceSequenceId => _maintenanceSequence + 1; + + /// + /// Sends MOVING to one client, or to every client that opted in when + /// is null. A null is the documented no-address form, which a client + /// must handle whether or not it asked for none. + /// + /// The number of clients the notification was sent to. + public int SendMoving(RedisClient client, int timeSeconds, EndPoint newEndpoint, int? sequenceId = null) + { + var recipients = MovingClosesConnection ? new List() : null; + Action onSent = recipients is null ? null : recipients.Add; + var count = Send(client, MaintenanceNotificationKind.Moving, timeSeconds, sequenceId, newEndpoint, null, onSent); + + if (recipients is { Count: > 0 }) + { + // The half of MOVING that defines it: the socket goes away - and it takes every other + // connection to the same node with it (see MovingClosesConnection). + var delay = MovingCloseDelay ?? TimeSpan.FromSeconds(Math.Max(timeSeconds, 0)) + MeasuredCloseSlack; + _ = CloseAfterAsync(CollectSiblings(recipients), delay); + } + + return count; + } + + /// + /// Whether then closes the affected connections, as a real proxy does. + /// + /// + /// Note the blast radius is the *node*, not the connection. Measured on RS (2026-08-28) with four + /// connections to one node differing only in handshake - two opted in, one RESP3 without the opt-in, + /// one RESP2 - all four closed simultaneously, and only the two that had opted in were warned. So this + /// is endpoint retirement rather than socket recycling, and a connection that did not opt in gets no + /// warning at all before it dies: an argument for opting in on every connection rather than one. + /// + public bool MovingClosesConnection { get; set; } + + /// + /// How much later than the announced window the close actually arrives, by default. + /// + /// + /// Measured at +3.4s and +1.6s against a declared 15s grace, i.e. the announced window behaves as a + /// floor with slack rather than a deadline. Tests should assert that a client acts *within* the window, + /// never that the socket survives to the end of it. + /// + public TimeSpan MeasuredCloseSlack { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// How long after MOVING the connections are closed; defaults to the announced window plus + /// , which is what a real proxy was measured doing. + /// + /// + /// Set it shorter than the announced window to exercise a proxy less generous than the one measured - + /// a client that treats the window as guaranteed rather than as a budget fails that case. It cannot + /// usefully be set to zero: the close would race the delivery of the notification itself, which is not + /// something any real timing produces. + /// + public TimeSpan? MovingCloseDelay { get; set; } + + /// + /// Expands the notified connections to every connection sharing a node with them. + /// + private List CollectSiblings(List notified) + { + var nodes = new HashSet(); + foreach (var client in notified) nodes.Add(client.Node); + + var all = new List(); + ForAllClients( + all, + (client, list) => + { + if (nodes.Contains(client.Node)) list.Add(client); + return 0; + }); + return all; + } + + private async Task CloseAfterAsync(List clients, TimeSpan delay) + { + if (delay > TimeSpan.Zero) await Task.Delay(delay).ConfigureAwait(false); + foreach (var client in clients) + { + Log($"[{client}] closing connection after MOVING"); + client.Kill(); + } + } + + /// + /// Sends one of the shard-scoped notifications. is a remaining-time + /// delta and may legitimately be zero or negative for a connection that arrived mid-window; pass + /// null to omit the element entirely, which is what a real server does for the *closing* + /// notifications - captured from Enterprise 8.6.2: + /// + /// >4 $12 FAILING_OVER :0 :2 $6 ["21"] + /// >3 $11 FAILED_OVER :1 $6 ["21"] + /// + /// + /// The number of clients the notification was sent to. + public int SendShardNotification(RedisClient client, MaintenanceNotificationKind kind, int? timeSeconds, string shardIds = null, int? sequenceId = null) + => Send(client, kind, timeSeconds, sequenceId, null, shardIds); + + /// + /// Sends a slot-scoped notification (SMIGRATING / SMIGRATED) carrying a slot list in the + /// contract's comma-and-range form, e.g. "123,456,789-1000". + /// + /// The number of clients the notification was sent to. + public int SendSlotNotification(RedisClient client, MaintenanceNotificationKind kind, string slots, int? sequenceId = null) + => Send(client, kind, null, sequenceId, null, slots); + + /// + /// Sends SMIGRATED in its nested form - [type, seq, [[source, target, slots], ...]] - + /// which is what the shipped clients read (see the topic README's prior art; go-redis reads exactly + /// this shape and redis-py models the same nesting). + /// + /// + /// Note the sender is not implicitly the source of anything: every node reports the same movements, so + /// a test can and should exercise a delta that does not involve this server at all. + /// + /// The number of clients the notification was sent to. + public int SendSlotMigrations(RedisClient client, MaintenanceNotificationKind kind, (string Source, string Target, string Slots)[] migrations, int? sequenceId = null) + { + // one sequence id for the notification, however many clients it is delivered to - it identifies the + // event, not the delivery, which is what a real server does and what client-side dedup relies on + var seq = sequenceId ?? Interlocked.Increment(ref _maintenanceSequence); + return Dispatch(client, Build, requireOptIn: true); + + TypedRedisValue Build() + { + // Rent at every level: Recycle() recurses, so a Standalone child inside a pooled parent gets + // handed to the pool it did not come from ("The buffer is not associated with this pool") + var frame = TypedRedisValue.Rent(3, out var span, RespPrefix.Push); + // bulk, not simple: that is what a real server sends. Captured from Enterprise 8.6.2: + // >3 $9 SMIGRATED :19 *1[ *3[ $20 $18 $9 ] ] + span[0] = TypedRedisValue.BulkString(GetName(kind)); + span[1] = TypedRedisValue.Integer(seq); + + var outer = TypedRedisValue.Rent(migrations.Length, out var outerSpan, RespPrefix.Array); + for (int i = 0; i < migrations.Length; i++) + { + var triplet = TypedRedisValue.Rent(3, out var inner, RespPrefix.Array); + inner[0] = TypedRedisValue.BulkString(migrations[i].Source); + inner[1] = TypedRedisValue.BulkString(migrations[i].Target); + inner[2] = TypedRedisValue.BulkString(migrations[i].Slots); + outerSpan[i] = triplet; + } + + span[2] = outer; + return frame; + } + } + + /// + /// Sends an arbitrary push frame to a client, for the cases a well-formed notification cannot express: + /// an unknown type, a malformed payload, extra trailing elements. + /// + /// The number of clients the frame was sent to. + public int SendRawPush(RedisClient client, params string[] parts) + { + return Dispatch(client, Build, requireOptIn: false); + + TypedRedisValue Build() + { + var frame = TypedRedisValue.Rent(parts.Length, out var span, RespPrefix.Push); + for (int i = 0; i < parts.Length; i++) + { + span[i] = TypedRedisValue.BulkString(parts[i]); + } + return frame; + } + } + + private int Send( + RedisClient client, + MaintenanceNotificationKind kind, + int? timeSeconds, + int? sequenceId, + EndPoint newEndpoint, + string extra, + Action onSent = null) + { + // [type, seqID, ...] - the sequence id is an integer, which is precisely why these frames could + // not be treated as pub/sub: element 1 is not a channel name + var seq = sequenceId ?? System.Threading.Interlocked.Increment(ref _maintenanceSequence); + + // the retention is server-wide and replaces rather than accumulates, as a real one does; note it + // records what was *sent*, so a test arranges it by sending the completion, not by poking state + if (IsRetainedCompletion(kind)) _retainedCompletion = (kind, seq, extra); + + return Dispatch(client, Build, requireOptIn: true, onSent); + + TypedRedisValue Build() => BuildShardNotification(kind, timeSeconds, seq, newEndpoint, extra); + } + + private static TypedRedisValue BuildShardNotification( + MaintenanceNotificationKind kind, + int? timeSeconds, + long seq, + EndPoint newEndpoint, + string extra) + { + int count = 2 + (timeSeconds.HasValue ? 1 : 0) + (newEndpoint is not null || kind == MaintenanceNotificationKind.Moving ? 1 : 0) + (extra is not null ? 1 : 0); + var frame = TypedRedisValue.Rent(count, out var span, RespPrefix.Push); + + int index = 0; + span[index++] = TypedRedisValue.BulkString(GetName(kind)); // bulk, as a real server sends + span[index++] = TypedRedisValue.Integer(seq); + if (timeSeconds.HasValue) span[index++] = TypedRedisValue.Integer(timeSeconds.GetValueOrDefault()); + if (kind == MaintenanceNotificationKind.Moving) + { + // null rather than absent when there is no address: the client must cope with both + span[index++] = newEndpoint is null + ? TypedRedisValue.BulkString(RedisValue.Null) + : TypedRedisValue.BulkString(Format.ToString(newEndpoint)); + } + else if (newEndpoint is not null) + { + span[index++] = TypedRedisValue.BulkString(Format.ToString(newEndpoint)); + } + if (extra is not null) span[index] = TypedRedisValue.BulkString(extra); + + return frame; + } + + /// + /// Sends a notification to one client or to every opted-in client, building the frame per recipient. + /// + /// + /// The factory is called once per recipient rather than the frame being built once and shared, because + /// each client's write loop recycles what it wrote: a shared frame is returned to the pool by the first + /// writer and the second one faults with "Array element cannot be nil", killing that connection. Every + /// broadcast therefore used to reach exactly one client, which is invisible in a single-node test and + /// silently made multi-node fan-out untestable. + /// + private int Dispatch(RedisClient client, Func frameFactory, bool requireOptIn, Action onSent = null) + { + if (client is not null) + { + client.AddOutbound(frameFactory()); + onSent?.Invoke(client); + return 1; + } + + // a real server sends only to connections that asked, so sending to all means all *opted-in*. + // Counting the sends rather than the clients visited: the Action overload of ForAllClients returns + // one per client regardless, which would report every connection as a recipient + return ForAllClients( + requireOptIn, + (target, gated) => + { + if (gated && !target.MaintenanceNotifications) return 0; + target.AddOutbound(frameFactory()); + onSent?.Invoke(target); + return 1; + }); + } + } +} diff --git a/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs b/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs index 7778ed63b..1498ce1c0 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.PubSub.cs @@ -62,7 +62,9 @@ protected virtual TypedRedisValue SPublish(RedisClient client, in RedisRequest r var slot = ServerSelectionStrategy.GetClusterSlot((byte[])channel); if (!node.HasSlot(slot)) KeyMovedException.Throw(slot); - PublishPair pair = new(channel, request.GetValue(2)); + // note the node: without it pair.Node is null, ReferenceEquals never matches, and sharded publish + // silently delivers to nobody - which is how it behaved until 2026-08-26 + PublishPair pair = new(channel, request.GetValue(2), node); int count = ForAllClients(pair, static (client, pair) => ReferenceEquals(client.Node, pair.Node) ? client.Publish(pair.Channel, pair.Value) : 0); return TypedRedisValue.Integer(count); @@ -280,6 +282,38 @@ private Regex BuildGlob(RedisChannel channel) return new Regex(re, RegexOptions.CultureInvariant); } + /// + /// Drops this client's sharded subscriptions whose channel hashes into , + /// pushing the unsolicited sunsubscribe a real server sends when a slot moves away. + /// + /// How many subscriptions were dropped. + internal int UnsubscribeMigratedSlot(int hashSlot) + { + var subs = SubscriptionsIfAny; + if (subs is null) return 0; + + List affected = null; + lock (subs) + { + foreach (var pair in subs) + { + var channel = pair.Key; + if (channel.IsSharded && ServerSelectionStrategy.GetClusterSlot((byte[])channel) == hashSlot) + { + (affected ??= new()).Add(channel); + } + } + } + + if (affected is null) return 0; + foreach (var channel in affected) + { + Unsubscribe(channel); // removes it *and* sends the push, exactly as a client-issued one would + } + + return affected.Count; + } + internal void Unsubscribe(RedisChannel channel) { var subs = SubscriptionsIfAny; diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 409cbd31c..a319ea303 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -112,6 +112,12 @@ public bool Migrate(int hashSlot, EndPoint to) throw new KeyNotFoundException($"Unable to remove slot {hashSlot} from old owner"); } target.AddSlot(hashSlot); + + if (NotifyOnMigrate) + { + AnnounceMigration(hashSlot, pair.Value, target); + } + return true; } } @@ -311,6 +317,38 @@ public EndPoint AddEmptyNode(EndPoint endpoint, NodeFlags flags = NodeFlags.None return endpoint; } + /// + /// Removes a node from the cluster entirely, as CLUSTER FORGET does: it stops appearing in + /// CLUSTER SLOTS and in CLUSTER NODES, and any name it answered to stops resolving. + /// + /// + /// Refuses while it still owns slots, which is also what a real cluster does - a node has to have its + /// slots migrated away before it can be forgotten, and allowing it here would produce a topology with + /// unowned slots that says nothing useful about client behaviour. + /// + /// This exists to distinguish the two conditions a client has to tell apart: a node that currently + /// serves nothing is still a cluster member and may be given slots again, whereas a node that has + /// *left* is one whose connection should be given up. + /// + /// + public bool RemoveNode(EndPoint endpoint) + { + if (endpoint is null) throw new ArgumentNullException(nameof(endpoint)); + if (!_nodes.TryGetValue(endpoint, out var node)) return false; + if (node.HasAnySlot) throw new InvalidOperationException($"Node still owns slots: {Format.ToString(endpoint)}"); + + if (!_nodes.TryRemove(endpoint, out _)) return false; + + // and every alias that pointed at it, or a stale name would resolve to a node that is gone + foreach (var pair in _aliases) + { + if (ReferenceEquals(pair.Value, node)) _aliases.TryRemove(pair.Key, out _); + } + + Log($"node removed from the cluster: {Format.ToString(endpoint)}"); + return true; + } + public EndPoint AddEmptyNode(NodeFlags flags = NodeFlags.None) { EndPoint endpoint; @@ -694,6 +732,99 @@ protected virtual TypedRedisValue ClientReply(RedisClient client, in RedisReques return TypedRedisValue.OK; } + /// + /// How this server answers CLIENT MAINT_NOTIFICATIONS. Real servers vary: Enterprise supports + /// it, OSS and Valkey and Garnet do not, and Enterprise can have the feature flag off - so a client + /// must cope with all three, and a test must be able to ask for all three. + /// + public enum MaintenanceNotificationSupport + { + /// Accept the opt-in and reply +OK. + Supported = 0, + + /// Reply with an error, as a server that has never heard of the subcommand does. + UnknownSubcommand, + + /// Reply with an error, as a server whose feature flag is off does. + Disabled, + } + + /// + /// How this server answers the maintenance-notification opt-in; + /// by default, so that any test may opt in and see it accepted. + /// + public MaintenanceNotificationSupport MaintenanceNotifications { get; set; } + + /// + /// The moving-endpoint-type values this server accepts. + /// + /// + /// Configurable because a deployment need not offer all of them - an internal address is meaningless to + /// a client outside the network, and a deployment with no public DNS cannot offer an external FQDN. A + /// client that asks for one it cannot have gets an error, and that is an ordinary refusal: the feature + /// stays off and the connection carries on. + /// + public string[] SupportedMovingEndpointTypes { get; set; } = + ["internal-ip", "internal-fqdn", "external-ip", "external-fqdn", "none"]; + + // CLIENT MAINT_NOTIFICATIONS [parameter value ...], where parameter names follow a + // $type-$setting convention and moving-endpoint-type is the only one defined so far. A bare ON is + // explicitly valid and means "use the server defaults", so the parameter list is optional and + // unrecognized parameters are an error rather than something to ignore - the client is asking the + // server to do something specific, and silently not doing it would be worse than refusing + private static bool IsKeyword(in RedisRequest request, int index, string keyword) + => string.Equals(request.GetString(index), keyword, StringComparison.OrdinalIgnoreCase); + + [RedisCommand(-3, nameof(RedisCommand.CLIENT), "maint_notifications", LockFree = true)] + protected virtual TypedRedisValue ClientMaintNotifications(RedisClient client, in RedisRequest request) + { + switch (MaintenanceNotifications) + { + case MaintenanceNotificationSupport.UnknownSubcommand: + return request.UnknownSubcommandOrArgumentCount(); + case MaintenanceNotificationSupport.Disabled: + return TypedRedisValue.Error("ERR maintenance notifications are disabled on this server"); + } + + // keywords are matched case-insensitively, as a real server does; clients differ here (go-redis + // sends lowercase, we send the same uppercase form we use for every other keyword), and a fake + // that only accepted one of them would fail a client for something a real server allows + bool on; + if (IsKeyword(request, 2, "on")) on = true; + else if (IsKeyword(request, 2, "off")) on = false; + else return TypedRedisValue.Error("ERR syntax error"); + + string movingEndpointType = null; + for (int i = 3; i < request.Count; i += 2) + { + if (i + 1 >= request.Count) return TypedRedisValue.Error("ERR syntax error"); + + if (IsKeyword(request, i, "moving-endpoint-type")) + { + movingEndpointType = request.GetString(i + 1)?.ToLowerInvariant(); + if (movingEndpointType is null || Array.IndexOf(SupportedMovingEndpointTypes, movingEndpointType) < 0) + { + return TypedRedisValue.Error($"ERR unsupported moving-endpoint-type '{movingEndpointType}'"); + } + } + else + { + return TypedRedisValue.Error($"ERR unknown parameter '{request.GetString(i)}'"); + } + } + + client.MaintenanceNotifications = on; + client.MovingEndpointType = on ? movingEndpointType : null; + client.MaintenanceNotificationOptInCount++; + if (on) + { + OnMaintenanceOptIn(); + ReplayRetainedCompletion(client); // the catch-up channel, immediately after this +OK + } + Log($"[{client}] maintenance notifications {(on ? "on" : "off")}, moving-endpoint-type: {movingEndpointType ?? "(server default)"}"); + return TypedRedisValue.OK; + } + [RedisCommand(2, nameof(RedisCommand.CLIENT), "id", LockFree = true)] protected virtual TypedRedisValue ClientId(RedisClient client, in RedisRequest request) => TypedRedisValue.Integer(client.Id); @@ -1003,6 +1134,12 @@ public Node(RedisServer server, EndPoint endpoint, NodeFlags flags) } public void UpdateSlots(SlotRange[] slots) => _slots = slots; + + /// + /// Whether this node serves anything at all; note a null slot set means "all slots" + /// (the single-node default), not "none". + /// + public bool HasAnySlot => _slots is null || _slots.Length != 0; public ReadOnlySpan Slots => _slots ?? SlotRange.SharedAllSlots; public bool CheckCrossSlot => _server.CheckCrossSlot;