From 41d68f3f838ccebf9dc8e4aac878181e382223b9 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 27 Oct 2026 19:27:31 +0000 Subject: [PATCH 1/3] client cache: CacheTrackingMode.None and CacheTrackingMode.KeyspaceNotifications Two invalidation strategies that do not use CLIENT TRACKING, and so do not need RESP3: - None: nothing is negotiated; entries are retired by TimeToLive and by local writes alone. Prefixes, if given, only bound what is cached. - KeyspaceNotifications: the cache subscribes to __keyspace@*__:{prefix}* per prefix (or __keyspace@*__:* without prefixes) and invalidates the named key on any event. Multi-node channels, so cluster subscriptions go to every node, as keyspace notifications require. If the server has notifications disabled the subscription is silent and the cache behaves as None; that is documented, not checked. Prefixes are a hard boundary on what is cached (as under Broadcast) and must not contain glob characters. The handshake skips CLIENT TRACKING (and its RESP3 requirement) for both. The subscription is made after connect in both connect paths; a failure to establish it is logged rather than failing the connect, since it is registered first and restored with the subscription connection. Claude-Session: https://claude.ai/code/session_01DUpQ8hTYNcGjwf1eA7R6Ss --- docs/ClientSideCaching.md | 10 +- .../Caching/CacheOptions.cs | 46 +++- .../Caching/CacheTrackingMode.cs | 63 +++++ .../ConnectionMultiplexer.ClientCache.cs | 61 +++++ .../ConnectionMultiplexer.cs | 2 + .../Contexts/RespHandshake.cs | 4 +- src/StackExchange.Redis/LoggerExtensions.cs | 6 + .../PublicAPI/PublicAPI.Unshipped.txt | 2 + .../RespCacheInvalidationStrategyTests.cs | 221 ++++++++++++++++++ 9 files changed, 403 insertions(+), 12 deletions(-) create mode 100644 src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs create mode 100644 tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs diff --git a/docs/ClientSideCaching.md b/docs/ClientSideCaching.md index 50415ab63..87c28999d 100644 --- a/docs/ClientSideCaching.md +++ b/docs/ClientSideCaching.md @@ -32,7 +32,7 @@ With `Prefixes` set, the prefixes decide what is cached: a read inside them is c Requirements: -- **RESP3.** Invalidations arrive as out-of-band pushes, which only RESP3 can deliver on the same connection. With a cache configured and RESP2, connecting **fails** with a message saying so, rather than quietly running without invalidation. See [Using RESP3](Resp3). +- **RESP3.** Invalidations arrive as out-of-band pushes, which only RESP3 can deliver on the same connection. With a cache configured and RESP2, connecting **fails** with a message saying so, rather than quietly running without invalidation. See [Using RESP3](Resp3). The two modes that do not use `CLIENT TRACKING` - `KeyspaceNotifications` and `None`, below - do not need RESP3. - **A server that supports `CLIENT TRACKING`** (Redis 6.0 or later, or a compatible server). If the server refuses it - including a managed service that disables the command - the connection fails, for the same reason: a cache that is never invalidated is worse than no cache. The cache is configured in code only - it is not part of the connection string, because its settings are correctness choices that deserve to be read before they are set. @@ -196,18 +196,20 @@ Under `MaxBytes`, replies are stored in large slabs in the order they arrive and - **`TrackingByPrefix`** (the default when `Prefixes` are set): `Prefixes` decide for the calls that pass no flag. A read whose keys all fall under a prefix is treated as if it had said `CommandFlags.CacheOptIn`: it goes out behind `CLIENT CACHING YES`, the server tracks it, and it is stored on its first miss. Any other read goes to the server as it would without a cache. The flags still win: `CacheOptIn` on a read outside the prefixes caches it, `CacheOptOut` on a read inside them does not. Negotiated as `CLIENT TRACKING ON OPTIN`; the list is **not** sent to the server (the server only accepts `PREFIX` with `BCAST`), so the server tracks exactly the keys this client holds. The price is an extra frame and reply per miss inside the prefixes, which is the traffic the cache exists to shrink. Prefixes are `RedisKey` values, so a byte prefix is as good as a string one, and a key made with `WithKeyPrefix` can be given as it is. They must not overlap one another (`"app:"` and `"app:user:"` together are rejected), and they are compared with the key as sent - so if you build keys with `WithKeyPrefix`, the prefix must include that. - **`TrackingOptIn`**: opt-in by flag alone. Negotiated as `CLIENT TRACKING ON OPTIN`: the server tracks nothing, and nothing is cached, unless a call says so with `CommandFlags.CacheOptIn` - that read goes out behind `CLIENT CACHING YES`, is tracked, and is stored on its first miss. Every other read goes to the server as it would without a cache. For "cache these few hot keys and leave everything else alone" when the keys are not a prefix; the price is an extra frame and an extra reply per opted-in miss. `Prefixes` cannot be combined with it - that is `TrackingByPrefix`. - **`Broadcast`**: the server announces every change to every key matching your `Prefixes`, whoever made it, and keeps no record of what you read. Cheap for the server; the cost is on the client, which hears about keys it never cached, and that cost is set by how busy the server is rather than by anything this client does - which is why prefixes no longer select this mode by default. **Set `Prefixes`** if you choose it: with none, this connection hears about every write to every key in the instance. Keys outside the prefixes are simply not cached, and here the list is a hard boundary: the server was never asked about those keys, so no flag can opt one in. The per-call flags keep only their local meaning: the server has no per-client table to opt in to or out of. +- **`KeyspaceNotifications`**: no `CLIENT TRACKING` at all; the cache subscribes to [keyspace notifications](KeyspaceNotifications) instead - `__keyspace@*__:{prefix}*` once per entry in `Prefixes`, or `__keyspace@*__:*` without them - and invalidates the key named by any event. For servers or proxies that do not offer `CLIENT TRACKING` but do offer pub/sub, and it works over RESP2. **The server must have notifications enabled** (`notify-keyspace-events` including `K` and the event classes that can change what you cache; `KA` covers everything, expiry and eviction included). This is not checked: with notifications off the subscription is silent and the cache behaves as `None`. Like `Broadcast`, it hears about every matching change by anyone, so set `Prefixes` on a busy server; they are a hard boundary on what is cached, and must not contain glob characters (`*`, `?`, `[`). In a cluster each node raises notifications only for its own keys, so the subscriptions are made on every node. Weaker than tracking in two further ways: a `FLUSHDB`/`FLUSHALL` by another client raises no per-key event, and pub/sub does not replay what was missed while the subscription was down (losing a connection flushes the cache, as in every mode). `TimeToLive` bounds both. +- **`None`**: no server-assisted invalidation. Entries are retired by `TimeToLive` and by writes made through this multiplexer, and nothing else: a write by another client is served stale until the entry ages out, so choose the lifetime as the staleness you are prepared to accept. Nothing is negotiated, so it works over RESP2 and against any server. `Prefixes`, if set, only bound what is cached. `CacheTrackingMode.Default` (the value you get by not setting it) means "the library's choice", resolved as above today; name a mode explicitly if you depend on it. What the two flags do, by mode: -| | `Tracking` | `TrackingByPrefix` | `TrackingOptIn` | `Broadcast` | +| | `Tracking` | `TrackingByPrefix` | `TrackingOptIn` | `Broadcast`, `KeyspaceNotifications`, `None` | |---|---|---|---|---| | no flag | cached; tracked | inside `Prefixes`: cached, stored on first miss, `CLIENT CACHING YES` sent; outside: not cached, not tracked | not cached; not tracked | cached if inside `Prefixes` | | `CacheOptIn` | cached, stored on first miss | cached, stored on first miss; `CLIENT CACHING YES` sent - inside or outside `Prefixes` | cached, stored on first miss; `CLIENT CACHING YES` sent | cached if inside `Prefixes`, stored on first miss | | `CacheOptOut` | not cached; `CLIENT CACHING NO` sent | not cached | not cached | not cached | -Passing both flags is rejected. Neither flag can make an uncacheable command cacheable (a write, a keyless read, a random or time-based read). In `Broadcast` mode `CacheOptIn` does not bypass `Prefixes`; in `TrackingByPrefix` mode it does, because there the list is only the policy for calls that passed no flag. +Passing both flags is rejected. Neither flag can make an uncacheable command cacheable (a write, a keyless read, a random or time-based read). In `Broadcast`, `KeyspaceNotifications` and `None` modes `CacheOptIn` does not bypass `Prefixes`; in `TrackingByPrefix` mode it does, because there the list is only the policy for calls that passed no flag. ### Invalidating by hand @@ -225,7 +227,7 @@ The same object carries the cache's counters - `Count`, `Bytes`, `Stored`, `Evic ### Connection groups -With a [connection group](Failover) (geo-failover, Active-Active), the cache is configured once, on `MultiGroupOptions.ClientCache`, and every member inherits it; a member whose own `ConfigurationOptions.ClientCache` is set is refused when the group connects. Each member still needs `Protocol = RedisProtocol.Resp3`. +With a [connection group](Failover) (geo-failover, Active-Active), the cache is configured once, on `MultiGroupOptions.ClientCache`, and every member inherits it; a member whose own `ConfigurationOptions.ClientCache` is set is refused when the group connects. Each member still needs `Protocol = RedisProtocol.Resp3`, unless the mode is `KeyspaceNotifications` or `None`. ```csharp MultiGroupOptions options = new MultiGroupOptions.Builder diff --git a/src/StackExchange.Redis/Caching/CacheOptions.cs b/src/StackExchange.Redis/Caching/CacheOptions.cs index 5d20bb5cd..40c16f7ac 100644 --- a/src/StackExchange.Redis/Caching/CacheOptions.cs +++ b/src/StackExchange.Redis/Caching/CacheOptions.cs @@ -67,6 +67,14 @@ public sealed class CacheOptions var explicitMode => explicitMode, }; + /// Whether the resolved mode is negotiated with CLIENT TRACKING at the handshake. + /// + /// False for , which negotiates nothing, and for + /// , which subscribes instead - and so for both, RESP3 is + /// not required. + /// + internal bool UsesClientTracking => ResolvedTrackingMode is not (CacheTrackingMode.None or CacheTrackingMode.KeyspaceNotifications); + /// How entries behave, unless a caller says otherwise. public CachePolicy DefaultPolicy { get; init; } = CachePolicy.Default; @@ -269,7 +277,7 @@ public int EvictionSampleSize /// total one. /// /// - /// Two modes read it. Under it goes to the server as + /// Four modes read it. Under it goes to the server as /// PREFIX arguments and filters what is announced, as well as what is cached. Under /// it stays here and decides which unflagged reads /// are opted in (sent behind CLIENT CACHING YES, tracked, stored); the per-call flags override @@ -281,6 +289,12 @@ public int EvictionSampleSize /// selects . /// Either mistake is raised when the cache is built rather than at the handshake. /// + /// + /// Under it is the set of keyspace channels subscribed + /// to, one pattern per prefix, and the same hard boundary on what is cached as under + /// . Under it only bounds what + /// is cached; nothing is sent. + /// /// public IReadOnlyList Prefixes { @@ -316,8 +330,8 @@ public IReadOnlyList Prefixes /// Whether restricts what may be cached. /// - /// Only ever true in and - /// , the two modes that accept a list: under + /// Never true in or , + /// the two modes that refuse a list: under /// the server announces exactly what we read, so there is no /// such thing as an untracked key and nothing for the gate to refuse. /// @@ -335,15 +349,33 @@ public IReadOnlyList Prefixes internal void Validate() { var mode = ResolvedTrackingMode; - if (HasPrefixes && mode is not (CacheTrackingMode.Broadcast or CacheTrackingMode.TrackingByPrefix)) + if (HasPrefixes && mode is (CacheTrackingMode.Tracking or CacheTrackingMode.TrackingOptIn)) { throw new ArgumentException( - $"Cache prefixes require {nameof(CacheTrackingMode)}.{nameof(CacheTrackingMode.Broadcast)}" - + $" or {nameof(CacheTrackingMode)}.{nameof(CacheTrackingMode.TrackingByPrefix)};" - + $" {TrackingMode} announces the keys that were read, so there is nothing to filter.", + $"Cache prefixes require {nameof(CacheTrackingMode)}.{nameof(CacheTrackingMode.Broadcast)}," + + $" {nameof(CacheTrackingMode.TrackingByPrefix)}, {nameof(CacheTrackingMode.KeyspaceNotifications)}" + + $" or {nameof(CacheTrackingMode.None)}; {TrackingMode} announces the keys that were read, so there" + + " is nothing to filter.", nameof(Prefixes)); } + if (mode == CacheTrackingMode.KeyspaceNotifications) + { + // each prefix becomes the pattern __keyspace@*__:{prefix}*, and a glob character in it would match + // something other than the prefix - a wider set would be harmless, but a narrower one ("[ab]" is one + // byte, not four) leaves cached keys nobody announces + foreach (var prefix in _prefixBytes) + { + if (prefix.AsSpan().IndexOfAny((byte)'*', (byte)'?', (byte)'[') >= 0) + { + throw new ArgumentException( + $"{nameof(CacheTrackingMode)}.{nameof(CacheTrackingMode.KeyspaceNotifications)} subscribes to" + + " a pattern per prefix, so a prefix must not contain '*', '?' or '['.", + nameof(Prefixes)); + } + } + } + if (mode == CacheTrackingMode.TrackingByPrefix && !HasPrefixes) { throw new ArgumentException( diff --git a/src/StackExchange.Redis/Caching/CacheTrackingMode.cs b/src/StackExchange.Redis/Caching/CacheTrackingMode.cs index d8e34afca..8ceb4706e 100644 --- a/src/StackExchange.Redis/Caching/CacheTrackingMode.cs +++ b/src/StackExchange.Redis/Caching/CacheTrackingMode.cs @@ -155,5 +155,68 @@ public enum CacheTrackingMode /// /// TrackingByPrefix = 4, + + /// + /// No server-assisted invalidation: an entry is retired by , and by + /// writes made through this connection, and nothing else. + /// + /// + /// + /// A statement that staleness up to is acceptable. Nothing is + /// negotiated with the server - no CLIENT TRACKING, no subscription - so a write by any other client + /// is never heard about, and the entry it made wrong is served until it ages out. That is the whole bound, + /// so set the lifetime with that in mind rather than leaving it at the default because it was there. + /// + /// + /// What it buys: no server-side cost at all, and no requirement on the server or the protocol - it works + /// over RESP2, and against servers or proxies that do not implement CLIENT TRACKING. Writes made + /// through this multiplexer still invalidate what it holds, as they do in every mode. + /// + /// + /// , when given, say what may be cached: a read whose keys do not all fall + /// under one is sent and not stored. There is no server half to them here. + /// + /// + None = 5, + + /// + /// Invalidation from keyspace notifications (__keyspace@*__: pub/sub) rather than + /// CLIENT TRACKING. If the server does not have keyspace notifications enabled, this operates as + /// : entries are retired by alone. + /// + /// + /// + /// For servers - or deployments - where CLIENT TRACKING is not available, but pub/sub is. It needs + /// no RESP3: notifications arrive on the subscription connection, which RESP2 already has. The cache + /// subscribes to __keyspace@*__:{prefix}* once per entry in , or + /// to __keyspace@*__:* when there are none, and invalidates the key named by any event on it. + /// + /// + /// Enabling notifications is the server's business, and is not checked. They are off by default + /// (notify-keyspace-events ""); the cache needs K plus the event classes that can change what + /// it holds - KA covers all of them, and includes expiry (x) and eviction (e). With them + /// off the subscription is simply silent, and nothing distinguishes that from a server where nothing is + /// being written; hence "operates as ", which is the honest description of the outcome. + /// + /// + /// Weaker than tracking in three ways. Like , the server announces every + /// matching change by anyone, not just keys this client read - so scope it with + /// on a busy server. A flush (FLUSHDB/FLUSHALL) by another + /// client emits no per-key event, so it is not seen. And pub/sub is at-most-once: a notification lost with + /// the subscription connection is not replayed, which is why losing any connection flushes the cache, as it + /// does in every mode. bounds what remains. + /// + /// + /// Cluster: keyspace notifications are raised by the node that owns the key and are not propagated + /// to other nodes, so the subscriptions are made on every node; the library's handling of keyspace + /// channels already does this. A prefix containing a glob character (*, ?, [) is + /// rejected, because it would mean something different as a pattern than as a prefix. + /// + /// + /// are a hard boundary here, as under : a key + /// outside them is never announced, so it is never cached. + /// + /// + KeyspaceNotifications = 6, } } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs b/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs new file mode 100644 index 000000000..8c5abe6a6 --- /dev/null +++ b/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs @@ -0,0 +1,61 @@ +using System; +using System.Threading.Tasks; +using StackExchange.Redis.Caching; + +namespace StackExchange.Redis; + +public partial class ConnectionMultiplexer +{ + /// + /// Under , subscribe the client-side cache to the keyspace + /// channels that invalidate it; nothing to do in any other mode. + /// + /// + /// + /// One pattern per prefix (__keyspace@*__:{prefix}*), or a single wildcard without prefixes. Every + /// database, because the cache keys its entries by database but invalidates by key alone - as a tracking push + /// does - so a notification from another database costs a miss, never a stale read. + /// + /// + /// Cluster: these are multi-node channels, so the subscriber makes them on every primary, which is + /// what keyspace notifications need - each node raises them only for the keys it owns. + /// + /// + /// Not fatal. The subscription is registered before it is sent, so it is restored with the + /// subscription connection whatever happens here; a failure to establish it now (pub/sub refused, say) is + /// logged, and leaves the cache retired by - which is what this mode + /// promises when the server is not raising notifications anyway. + /// + /// + internal async Task SubscribeCacheInvalidationAsync() + { + if (ClientCache is not { Options.ResolvedTrackingMode: CacheTrackingMode.KeyspaceNotifications } cache) return; + + // the channel is __keyspace@{db}__:{key}, and the key is what the cache matches on, as the bytes it was sent as + Action handler = (channel, _) => cache.OnInvalidate(RedisChannel.StripKeySpacePrefix(channel.Span)); + + ISubscriber subscriber = DefaultSubscriber; + var prefixes = cache.Options.PrefixBytes; + try + { + if (prefixes.Length == 0) + { + await subscriber.SubscribeAsync(RedisChannel.KeySpacePattern(RedisKey.Null), handler).ForAwait(); + } + else + { + var pending = new Task[prefixes.Length]; + for (var i = 0; i < prefixes.Length; i++) + { + pending[i] = subscriber.SubscribeAsync(RedisChannel.KeySpacePrefix(prefixes[i].AsSpan()), handler); + } + + await Task.WhenAll(pending).ForAwait(); + } + } + catch (Exception ex) + { + Logger?.LogWarningCacheSubscriptionFailed(ex, ex.Message); + } + } +} diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 73fea6af5..8e00a8af2 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -697,6 +697,7 @@ private static async Task ConnectImplAsync(ConfigurationO Interlocked.Increment(ref muxer._connectCompletedCount); await muxer.ConnectEndpointsAsync().ForAwait(); + await muxer.SubscribeCacheInvalidationAsync().ForAwait(); if (muxer.ServerSelectionStrategy.ServerType == ServerType.Sentinel) { @@ -824,6 +825,7 @@ private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configurat Interlocked.Increment(ref muxer._connectCompletedCount); muxer.ConnectEndpointsAsync().Wait(muxer.SyncConnectTimeout(true)); + muxer.SubscribeCacheInvalidationAsync().Wait(muxer.SyncConnectTimeout(true)); if (muxer.ServerSelectionStrategy.ServerType == ServerType.Sentinel) { diff --git a/src/StackExchange.Redis/Contexts/RespHandshake.cs b/src/StackExchange.Redis/Contexts/RespHandshake.cs index b74e26cd0..3f090dccc 100644 --- a/src/StackExchange.Redis/Contexts/RespHandshake.cs +++ b/src/StackExchange.Redis/Contexts/RespHandshake.cs @@ -585,7 +585,9 @@ internal static async Task PerformAsync( } } - if (clientCache is not null) + // None negotiates nothing, and KeyspaceNotifications subscribes instead (see + // ConnectionMultiplexer.SubscribeCacheInvalidationAsync); neither needs RESP3 + if (clientCache is { Options.UsesClientTracking: true }) { await EnableClientTrackingAsync(context, clientCache, protocol).ConfigureAwait(false); } diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index cc999dd66..31796641c 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -886,6 +886,12 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "A pub/sub delivery for {Channel} (subscription {Subscription}) matched no subscription and was dropped")] internal static partial void LogDebugDeliveryUnmatched(this ILogger logger, string? channel, string? subscription); + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 138, + Message = "The client-side cache could not subscribe to keyspace notifications; entries will be retired by their time-to-live alone until the subscription is restored: {ErrorMessage}")] + internal static partial void LogWarningCacheSubscriptionFailed(this ILogger logger, Exception exception, string errorMessage); + [LoggerMessage( Level = LogLevel.Warning, EventId = 130, diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index ef0fcf4be..f9f7db301 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -72,6 +72,8 @@ StackExchange.Redis.Caching.CachePolicy.TimeToLive.init -> void StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.CacheTrackingMode.Broadcast = 1 -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.CacheTrackingMode.Default = 0 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.KeyspaceNotifications = 6 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.None = 5 -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.CacheTrackingMode.Tracking = 2 -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.CacheTrackingMode.TrackingOptIn = 3 -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.CacheTrackingMode.TrackingByPrefix = 4 -> StackExchange.Redis.Caching.CacheTrackingMode diff --git a/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs b/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs new file mode 100644 index 000000000..e2905fc5f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs @@ -0,0 +1,221 @@ +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using StackExchange.Redis.Caching; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The invalidation strategies that do not use CLIENT TRACKING: , retired by +/// lifetime and local writes alone, and , fed by pub/sub. +/// +/// +/// Both run over RESP2 here on purpose: not needing RESP3 is half of why either exists. The test servers are +/// configured with notify-keyspace-events AKE; nothing here changes that, since it is server-wide. +/// +public class RespCacheInvalidationStrategyTests(ITestOutputHelper output) : TestBase(output) +{ + private static async Task WaitFor(Func> condition, int millis = 3000) + { + var watch = Stopwatch.StartNew(); + while (watch.ElapsedMilliseconds < millis) + { + if (await condition()) return true; + await Task.Delay(25); + } + + return await condition(); + } + + private async Task ConnectAsync(string configuration, CacheOptions cacheOptions) + { + var options = ConfigurationOptions.Parse(configuration); + options.Protocol = RedisProtocol.Resp2; + options.ClientCache = cacheOptions; + try + { + return await ConnectionMultiplexer.ConnectAsync(options, Writer); + } + catch (RedisConnectionException ex) + { + Assert.Skip("Unable to connect to server: " + ex.Message); + throw; + } + } + + /// Read a key until the reply actually lands in the cache. + /// + /// As under BCAST (see RespCacheInvalidationTests.PrimeAsync): the notification for the setup + /// write can overtake the reply being filled from, and the fill is then rightly refused as raced. Not in + /// , where nothing arrives, so one read is enough there. + /// + private static async Task PrimeAsync(IDatabase db, RespClientCache cache, RedisKey key, string expected) + { + var stored = cache.Stored; + Assert.True( + await WaitFor(async () => + { + Assert.Equal(expected, (string?)await db.Strings.GetAsync(key)); + return cache.Stored > stored; + }), + $"'{key}' was never cached: refusedNotTracked={cache.RefusedNotTracked} raced={cache.RefusedRaced}"); + } + + [Fact] + public void PrefixesAreAcceptedByTheModesThatReadThem() + { + foreach (var mode in new[] { CacheTrackingMode.None, CacheTrackingMode.KeyspaceNotifications }) + { + using var cache = new RespClientCache(new CacheOptions { TrackingMode = mode, Prefixes = ["a:", "b:"] }); + Assert.True(cache.Options.HasPrefixes); + Assert.False(cache.Options.UsesClientTracking); + } + } + + [Fact] + public void TrackingModesStillNegotiateClientTracking() + { + foreach (var mode in new[] { CacheTrackingMode.Default, CacheTrackingMode.Broadcast, CacheTrackingMode.Tracking, CacheTrackingMode.TrackingOptIn }) + { + Assert.True(new CacheOptions { TrackingMode = mode }.UsesClientTracking, mode.ToString()); + } + } + + [Theory] + [InlineData("a*")] + [InlineData("a?")] + [InlineData("a[b]")] + public void KeyspacePrefixesMustNotBePatterns(string prefix) + { + var ex = Assert.Throws( + () => new RespClientCache(new CacheOptions { TrackingMode = CacheTrackingMode.KeyspaceNotifications, Prefixes = [prefix] })); + Assert.Equal(nameof(CacheOptions.Prefixes), ex.ParamName); + + // the same prefix means nothing special when nothing is subscribed + using var _ = new RespClientCache(new CacheOptions { TrackingMode = CacheTrackingMode.None, Prefixes = [prefix] }); + } + + /// + /// hears nothing from the server: a write elsewhere is served stale until the + /// lifetime retires it, while a write through this multiplexer is seen at once. + /// + [Fact] + public async Task NoneServesAnotherClientsWriteStaleButSeesItsOwn() + { + var me = Me(); + using var muxer = await ConnectAsync( + TestConfig.Current.PrimaryServerAndPort, + new CacheOptions { TrackingMode = CacheTrackingMode.None, Admission = CacheAdmission.OnFirstMiss }); + var cache = muxer.ClientCache; + Assert.NotNull(cache); + + using var other = await ConnectionMultiplexer.ConnectAsync(TestConfig.Current.PrimaryServerAndPort, Writer); + var writer = other.GetDatabase(); + var db = muxer.GetDatabase(); + RedisKey key = me; + + await writer.StringSetAsync(key, "v1"); + await PrimeAsync(db, cache, key, "v1"); + + await writer.StringSetAsync(key, "v2"); + await Task.Delay(200); // long enough for any invalidation to have arrived, were one coming + Assert.Equal("v1", (string?)await db.Strings.GetAsync(key)); + + await db.StringSetAsync(key, "v3"); + Assert.Equal("v3", (string?)await db.Strings.GetAsync(key)); + } + + /// + /// over RESP2: a write elsewhere invalidates through the + /// keyspace channel, with and without prefixes. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task KeyspaceNotificationsInvalidateAnotherClientsWrite(bool withPrefix) + { + var me = Me() + ":" + withPrefix + ":"; + using var muxer = await ConnectAsync( + TestConfig.Current.PrimaryServerAndPort, + new CacheOptions + { + TrackingMode = CacheTrackingMode.KeyspaceNotifications, + Admission = CacheAdmission.OnFirstMiss, + Prefixes = withPrefix ? [me] : [], + }); + await AssertInvalidatedAsync(muxer, TestConfig.Current.PrimaryServerAndPort, me, keyCount: 1); + } + + /// + /// The notification is raised only by the node that owns the key, so the subscription has to be on every + /// node; keys spread over the slots prove that it is. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task KeyspaceNotificationsInvalidateOnEveryClusterNode(bool withPrefix) + { + var me = Me() + ":" + withPrefix + ":"; + var config = GetClusterConfiguration(); + using var muxer = await ConnectAsync( + config, + new CacheOptions + { + TrackingMode = CacheTrackingMode.KeyspaceNotifications, + Admission = CacheAdmission.OnFirstMiss, + Prefixes = withPrefix ? [me] : [], + }); + Assert.SkipUnless(muxer.ServerSelectionStrategy.ServerType == ServerType.Cluster, "not a cluster"); + await AssertInvalidatedAsync(muxer, config, me, keyCount: 20); + } + + /// A key outside the prefixes is never announced, so it is never cached. + [Fact] + public async Task KeyspaceNotificationsRefuseAKeyOutsideThePrefixes() + { + var me = Me() + ":"; + using var muxer = await ConnectAsync( + TestConfig.Current.PrimaryServerAndPort, + new CacheOptions { TrackingMode = CacheTrackingMode.KeyspaceNotifications, Admission = CacheAdmission.OnFirstMiss, Prefixes = [me] }); + var cache = muxer.ClientCache; + Assert.NotNull(cache); + + var db = muxer.GetDatabase(); + RedisKey outside = "un" + me; + await db.StringSetAsync(outside, "v1"); + Assert.Equal("v1", (string?)await db.Strings.GetAsync(outside)); + Assert.Equal(1, cache.RefusedNotTracked); + } + + private async Task AssertInvalidatedAsync(ConnectionMultiplexer muxer, string configuration, string prefix, int keyCount) + { + var cache = muxer.ClientCache; + Assert.NotNull(cache); + + // somebody else, with no cache, whose writes we must hear about + using var other = await ConnectionMultiplexer.ConnectAsync(configuration, Writer); + var writer = other.GetDatabase(); + var db = muxer.GetDatabase(); + + var keys = new RedisKey[keyCount]; + for (var i = 0; i < keys.Length; i++) + { + keys[i] = prefix + i; + await writer.StringSetAsync(keys[i], "v1"); + await PrimeAsync(db, cache, keys[i], "v1"); + } + + foreach (var key in keys) + { + await writer.StringSetAsync(key, "v2"); + } + + foreach (var key in keys) + { + Assert.True( + await WaitFor(async () => (string?)await db.Strings.GetAsync(key) == "v2"), + $"the keyspace notification for '{key}' never invalidated it"); + } + } +} From 9f4dd9c57e1ab00ac2e98ebdc5447f23d102ea50 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 27 Oct 2026 20:10:41 +0000 Subject: [PATCH 2/3] client cache: CacheTrackingMode.None is the zero value None = 0 and Default = 1, the rest shifted up one (all unshipped). CacheOptions.TrackingMode is initialised to Default, so an unconfigured cache still gets the library's choice; only a zeroed enum is None. Claude-Session: https://claude.ai/code/session_01DUpQ8hTYNcGjwf1eA7R6Ss --- .../Caching/CacheAdmission.cs | 2 +- .../Caching/CacheOptions.cs | 2 +- .../Caching/CacheTrackingMode.cs | 71 ++++++++++--------- .../PublicAPI/PublicAPI.Unshipped.txt | 12 ++-- .../RespCacheInvalidationStrategyTests.cs | 9 +++ 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/StackExchange.Redis/Caching/CacheAdmission.cs b/src/StackExchange.Redis/Caching/CacheAdmission.cs index e5ad4f651..b974c7011 100644 --- a/src/StackExchange.Redis/Caching/CacheAdmission.cs +++ b/src/StackExchange.Redis/Caching/CacheAdmission.cs @@ -26,7 +26,7 @@ public enum CacheAdmission /// Let the library choose; today that is . /// /// - /// Zero means "no opinion", as it does for : an unset value is not a + /// Zero means "no opinion", as does for the tracking mode: an unset value is not a /// considered choice, and keeping the two apart lets the library change what it resolves to without /// overriding anybody who named a mode. Name one explicitly if you depend on its behaviour. /// diff --git a/src/StackExchange.Redis/Caching/CacheOptions.cs b/src/StackExchange.Redis/Caching/CacheOptions.cs index 40c16f7ac..c867c31df 100644 --- a/src/StackExchange.Redis/Caching/CacheOptions.cs +++ b/src/StackExchange.Redis/Caching/CacheOptions.cs @@ -46,7 +46,7 @@ public sealed class CacheOptions /// instead: the prefixes say what to cache, and the server still hears only about that. Read /// rather than this one when you need the mode that will actually be negotiated. /// - public CacheTrackingMode TrackingMode { get; init; } + public CacheTrackingMode TrackingMode { get; init; } = CacheTrackingMode.Default; /// /// with resolved to the mode it diff --git a/src/StackExchange.Redis/Caching/CacheTrackingMode.cs b/src/StackExchange.Redis/Caching/CacheTrackingMode.cs index 8ceb4706e..5d03926e4 100644 --- a/src/StackExchange.Redis/Caching/CacheTrackingMode.cs +++ b/src/StackExchange.Redis/Caching/CacheTrackingMode.cs @@ -22,17 +22,45 @@ namespace StackExchange.Redis.Caching /// public enum CacheTrackingMode { + /// + /// No server-assisted invalidation: an entry is retired by , and by + /// writes made through this connection, and nothing else. + /// + /// + /// + /// A statement that staleness up to is acceptable. Nothing is + /// negotiated with the server - no CLIENT TRACKING, no subscription - so a write by any other client + /// is never heard about, and the entry it made wrong is served until it ages out. That is the whole bound, + /// so set the lifetime with that in mind rather than leaving it at the default because it was there. + /// + /// + /// What it buys: no server-side cost at all, and no requirement on the server or the protocol - it works + /// over RESP2, and against servers or proxies that do not implement CLIENT TRACKING. Writes made + /// through this multiplexer still invalidate what it holds, as they do in every mode. + /// + /// + /// Zero, because it is the absence of something: what a zeroed ought to + /// mean. It is not what an unconfigured cache gets - starts as + /// - so leaving the mode unset never quietly turns invalidation off. + /// + /// + /// , when given, say what may be cached: a read whose keys do not all fall + /// under one is sent and not stored. There is no server half to them here. + /// + /// + None = 0, + /// /// Let the library choose; today that is , or /// when are given. /// /// /// - /// Zero means "no opinion", not a mode. Which mode is cheaper depends on whose resource is - /// scarce - the remarks on this type say the library cannot know that - and wiring a real mode to - /// zero says the opposite: that an unset value is a considered choice. It also makes the two - /// indistinguishable, so the library can never tell a caller who asked for a mode from one who - /// never thought about it, and can never change its mind without silently overriding the first. + /// "No opinion", not a mode, and what starts as. Which mode + /// is cheaper depends on whose resource is scarce - the remarks on this type say the library cannot know + /// that - and starting from a real mode says the opposite: that an unset value is a considered choice. It + /// also makes the two indistinguishable, so the library can never tell a caller who asked for a mode from + /// one who never thought about it, and can never change its mind without silently overriding the first. /// /// /// So the answer this resolves to may move - if server or client heuristics make one clearly @@ -40,7 +68,7 @@ public enum CacheTrackingMode /// if you need it pinned. /// /// - Default = 0, + Default = 1, /// /// BCAST: the server announces every changed key matching @@ -65,7 +93,7 @@ public enum CacheTrackingMode /// prefixes. /// /// - Broadcast = 1, + Broadcast = 2, /// /// CLIENT TRACKING ON OPTOUT: the server remembers which keys we read, and announces @@ -96,7 +124,7 @@ public enum CacheTrackingMode /// why NOLOOP is not simply switched on (design notes 6.13). /// /// - Tracking = 2, + Tracking = 3, /// /// CLIENT TRACKING ON OPTIN: the server remembers only the keys we explicitly ask it to, and @@ -120,7 +148,7 @@ public enum CacheTrackingMode /// server to opt out of. /// /// - TrackingOptIn = 3, + TrackingOptIn = 4, /// /// where decide for the reads that do @@ -154,30 +182,7 @@ public enum CacheTrackingMode /// where broadcasting is billed by the server's write rate. /// /// - TrackingByPrefix = 4, - - /// - /// No server-assisted invalidation: an entry is retired by , and by - /// writes made through this connection, and nothing else. - /// - /// - /// - /// A statement that staleness up to is acceptable. Nothing is - /// negotiated with the server - no CLIENT TRACKING, no subscription - so a write by any other client - /// is never heard about, and the entry it made wrong is served until it ages out. That is the whole bound, - /// so set the lifetime with that in mind rather than leaving it at the default because it was there. - /// - /// - /// What it buys: no server-side cost at all, and no requirement on the server or the protocol - it works - /// over RESP2, and against servers or proxies that do not implement CLIENT TRACKING. Writes made - /// through this multiplexer still invalidate what it holds, as they do in every mode. - /// - /// - /// , when given, say what may be cached: a read whose keys do not all fall - /// under one is sent and not stored. There is no server half to them here. - /// - /// - None = 5, + TrackingByPrefix = 5, /// /// Invalidation from keyspace notifications (__keyspace@*__: pub/sub) rather than diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index f9f7db301..342dac747 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -70,13 +70,13 @@ StackExchange.Redis.Caching.CachePolicy.RefreshAfter.init -> void StackExchange.Redis.Caching.CachePolicy.TimeToLive.get -> System.TimeSpan StackExchange.Redis.Caching.CachePolicy.TimeToLive.init -> void StackExchange.Redis.Caching.CacheTrackingMode -StackExchange.Redis.Caching.CacheTrackingMode.Broadcast = 1 -> StackExchange.Redis.Caching.CacheTrackingMode -StackExchange.Redis.Caching.CacheTrackingMode.Default = 0 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.Broadcast = 2 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.Default = 1 -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.CacheTrackingMode.KeyspaceNotifications = 6 -> StackExchange.Redis.Caching.CacheTrackingMode -StackExchange.Redis.Caching.CacheTrackingMode.None = 5 -> StackExchange.Redis.Caching.CacheTrackingMode -StackExchange.Redis.Caching.CacheTrackingMode.Tracking = 2 -> StackExchange.Redis.Caching.CacheTrackingMode -StackExchange.Redis.Caching.CacheTrackingMode.TrackingOptIn = 3 -> StackExchange.Redis.Caching.CacheTrackingMode -StackExchange.Redis.Caching.CacheTrackingMode.TrackingByPrefix = 4 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.None = 0 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.Tracking = 3 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.TrackingOptIn = 4 -> StackExchange.Redis.Caching.CacheTrackingMode +StackExchange.Redis.Caching.CacheTrackingMode.TrackingByPrefix = 5 -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.ClientCache StackExchange.Redis.Caching.ClientCache.Bytes.get -> long StackExchange.Redis.Caching.ClientCache.Count.get -> int diff --git a/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs b/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs index e2905fc5f..d46019dc6 100644 --- a/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs +++ b/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs @@ -73,6 +73,15 @@ public void PrefixesAreAcceptedByTheModesThatReadThem() } } + [Fact] + public void ZeroIsNoneButAnUnsetModeIsTheLibrarysChoice() + { + Assert.Equal(CacheTrackingMode.None, default(CacheTrackingMode)); + Assert.Equal(CacheTrackingMode.Default, new CacheOptions().TrackingMode); + Assert.Equal(CacheTrackingMode.Default, CacheOptions.Default.TrackingMode); + Assert.True(new CacheOptions().UsesClientTracking); + } + [Fact] public void TrackingModesStillNegotiateClientTracking() { From 34c03b1cd86396495f9f3efc1dc7f1c84751cfde Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 27 Oct 2026 20:39:31 +0000 Subject: [PATCH 3/3] client cache: Default falls back to keyspace notifications when tracking is unavailable Under CacheTrackingMode.Default only, a cache that cannot get CLIENT TRACKING switches to KeyspaceNotifications rather than failing the connection: no RESP3, CLIENT disabled in the command map (known at construction), or any server error answering CLIENT TRACKING. A named mode still fails loudly. The switch is cache-wide, one-way and at most once: the mode-dependent state (CLIENT CACHING frames, opt-in/by-prefix gates) is one immutable object swapped by CAS, so the frames a non-tracking server would reject stop at once. It flushes, starts the keyspace subscription (now idempotent and keyed off the mode in force), and logs a warning (event 139). When CONFIG GET is permitted and notify-keyspace-events raises nothing usable, a second warning says so (event 140) - a diagnostic, never a refusal. ClientCache.EffectiveTrackingMode (via GetClientCache()) reports the mode in force. Resp2WithACacheRefusesLoudly now names Tracking explicitly, since an unset mode over RESP2 falls back instead. Claude-Session: https://claude.ai/code/session_01DUpQ8hTYNcGjwf1eA7R6Ss --- docs/ClientSideCaching.md | 14 +- .../Caching/CacheOptions.cs | 11 ++ .../Caching/CacheTrackingMode.cs | 7 + .../Caching/ClientCache.cs | 9 ++ .../Caching/RespClientCache.cs | 119 +++++++++++++--- .../ConnectionMultiplexer.ClientCache.cs | 15 +- .../Contexts/RespHandshake.cs | 79 +++++++++-- src/StackExchange.Redis/LoggerExtensions.cs | 12 ++ .../PublicAPI/PublicAPI.Unshipped.txt | 1 + .../RespCacheInvalidationStrategyTests.cs | 134 ++++++++++++++++++ .../RespInProcTrackingTests.cs | 10 +- 11 files changed, 376 insertions(+), 35 deletions(-) diff --git a/docs/ClientSideCaching.md b/docs/ClientSideCaching.md index 87c28999d..99ef7b2a7 100644 --- a/docs/ClientSideCaching.md +++ b/docs/ClientSideCaching.md @@ -32,7 +32,7 @@ With `Prefixes` set, the prefixes decide what is cached: a read inside them is c Requirements: -- **RESP3.** Invalidations arrive as out-of-band pushes, which only RESP3 can deliver on the same connection. With a cache configured and RESP2, connecting **fails** with a message saying so, rather than quietly running without invalidation. See [Using RESP3](Resp3). The two modes that do not use `CLIENT TRACKING` - `KeyspaceNotifications` and `None`, below - do not need RESP3. +- **RESP3.** Invalidations arrive as out-of-band pushes, which only RESP3 can deliver on the same connection. With a tracking mode **named** and RESP2, connecting **fails** with a message saying so, rather than quietly running without invalidation. See [Using RESP3](Resp3). With the mode left unset, the cache instead falls back to keyspace notifications - see "Falling back", below. The two modes that do not use `CLIENT TRACKING` - `KeyspaceNotifications` and `None` - do not need RESP3. - **A server that supports `CLIENT TRACKING`** (Redis 6.0 or later, or a compatible server). If the server refuses it - including a managed service that disables the command - the connection fails, for the same reason: a cache that is never invalidated is worse than no cache. The cache is configured in code only - it is not part of the connection string, because its settings are correctness choices that deserve to be read before they are set. @@ -201,6 +201,18 @@ Under `MaxBytes`, replies are stored in large slabs in the order they arrive and `CacheTrackingMode.Default` (the value you get by not setting it) means "the library's choice", resolved as above today; name a mode explicitly if you depend on it. +#### Falling back + +Under `Default` only, when server-assisted tracking turns out to be unavailable, the cache switches to `KeyspaceNotifications` instead of failing the connection. That happens when: + +- the connection is not RESP3; +- `CLIENT` is disabled in the `CommandMap` (known before connecting); +- the server answers `CLIENT TRACKING` with any error - an older server, a proxy, an ACL. + +The switch is for the whole cache, happens at most once, and never reverts. It flushes the cache, starts the keyspace subscription, and logs a warning. If `CONFIG GET notify-keyspace-events` is permitted and shows that notifications are off, a second warning says so: the cache then behaves as `None`, retired by `TimeToLive` alone. That is a diagnostic, not a failure. `ClientCache.EffectiveTrackingMode` (from `GetClientCache()`) reports the mode actually in force. A mode named explicitly never falls back: it fails the connection as before. + +Keyspace notifications match prefixes as patterns, so a prefix containing `*`, `?` or `[` cannot be subscribed to; after a fallback such a cache logs the failed subscription and is retired by `TimeToLive` alone. + What the two flags do, by mode: | | `Tracking` | `TrackingByPrefix` | `TrackingOptIn` | `Broadcast`, `KeyspaceNotifications`, `None` | diff --git a/src/StackExchange.Redis/Caching/CacheOptions.cs b/src/StackExchange.Redis/Caching/CacheOptions.cs index c867c31df..4ff2df909 100644 --- a/src/StackExchange.Redis/Caching/CacheOptions.cs +++ b/src/StackExchange.Redis/Caching/CacheOptions.cs @@ -67,6 +67,17 @@ public sealed class CacheOptions var explicitMode => explicitMode, }; + /// + /// Whether a notify-keyspace-events value raises keyspace events that can invalidate anything. + /// + /// + /// K selects the keyspace channel, and on its own raises nothing: it needs at least one event class + /// with it. A is the alias for the classes that change a key. Not counted: n (a new key, which + /// has nothing cached) and m (a key miss, which is a read). + /// + internal static bool KeyspaceEventsInvalidate(string events) + => events.IndexOf('K') >= 0 && events.IndexOfAny(['A', 'g', '$', 'l', 's', 'h', 'z', 'x', 'e', 't', 'd']) >= 0; + /// Whether the resolved mode is negotiated with CLIENT TRACKING at the handshake. /// /// False for , which negotiates nothing, and for diff --git a/src/StackExchange.Redis/Caching/CacheTrackingMode.cs b/src/StackExchange.Redis/Caching/CacheTrackingMode.cs index 5d03926e4..98b0cb126 100644 --- a/src/StackExchange.Redis/Caching/CacheTrackingMode.cs +++ b/src/StackExchange.Redis/Caching/CacheTrackingMode.cs @@ -67,6 +67,13 @@ public enum CacheTrackingMode /// better, or if it starts depending on more of what else is configured. Name the mode explicitly /// if you need it pinned. /// + /// + /// It also falls back. When server-assisted tracking is unavailable - the connection is not RESP3, + /// CLIENT is disabled in the command map, or the server refuses CLIENT TRACKING - the cache + /// switches to rather than failing the connection, and logs that it has. + /// A mode named explicitly is a requirement, and fails instead. + /// reports the mode actually in force. + /// /// Default = 1, diff --git a/src/StackExchange.Redis/Caching/ClientCache.cs b/src/StackExchange.Redis/Caching/ClientCache.cs index 141e513d0..f878b9813 100644 --- a/src/StackExchange.Redis/Caching/ClientCache.cs +++ b/src/StackExchange.Redis/Caching/ClientCache.cs @@ -39,6 +39,15 @@ public sealed class ClientCache /// The options this cache was created with. public CacheOptions Options => _cache.Options; + /// The invalidation strategy actually in force. + /// + /// as resolved, unless it was and + /// server-assisted tracking turned out to be unavailable - no RESP3, CLIENT disabled, or the server + /// refusing CLIENT TRACKING - in which case it is . + /// It can change once, at any handshake, and never changes back. + /// + public CacheTrackingMode EffectiveTrackingMode => _cache.EffectiveTrackingMode; + /// Entries currently held, including ones invalidated or expired but not yet swept. public int Count => _cache.Count; diff --git a/src/StackExchange.Redis/Caching/RespClientCache.cs b/src/StackExchange.Redis/Caching/RespClientCache.cs index 51606217e..38f1642c3 100644 --- a/src/StackExchange.Redis/Caching/RespClientCache.cs +++ b/src/StackExchange.Redis/Caching/RespClientCache.cs @@ -107,15 +107,12 @@ public RespClientCache(CacheOptions? options = null, int keyCapacity = 256, Comm Options.Validate(); // settings that constrain one another; see CacheOptions.Validate _keys = new RespKeyTable(keyCapacity); - // the two CLIENT CACHING frames, rendered once through the command map (CLIENT can be renamed like - // anything else, and the handshake's CLIENT TRACKING honours the same map) and shared by every - // request that needs one: a prefix is a fixed buffer, never pooled, so sharing is free + // a disabled CLIENT is known before anything connects: under Default that is the first reason to fall + // back, and the only one that does not need a handshake to discover + var map = commandMap ?? CommandMap.Default; var mode = Options.ResolvedTrackingMode; - _optIn = mode == CacheTrackingMode.TrackingOptIn; - _byPrefix = mode == CacheTrackingMode.TrackingByPrefix; - _optOut = mode == CacheTrackingMode.Tracking; - if (_optIn || _byPrefix) _cachingYes = RenderClientCaching(commandMap ?? CommandMap.Default, "YES"u8); // both negotiate OPTIN - if (_optOut) _cachingNo = RenderClientCaching(commandMap ?? CommandMap.Default, "NO"u8); + if (CanFallBack && !map.IsAvailable(RedisCommand.CLIENT)) mode = CacheTrackingMode.KeyspaceNotifications; + _mode = new ModeState(mode, map); _slabSize = ChooseSlabSize(Options.MaxBytes); if (Options.ResolvedAdmission == CacheAdmission.OnRepeatedMiss) { @@ -134,10 +131,92 @@ public RespClientCache(CacheOptions? options = null, int keyCapacity = 256, Comm /// This cache as callers see it; one instance, created with the cache. See . internal ClientCache View { get; } - // _optIn: TrackingOptIn, where only a CacheOptIn read may be kept. _byPrefix: TrackingByPrefix, the same - // wire mode with Prefixes deciding for unflagged reads. _optOut: Tracking, where CacheOptOut has a wire half. - private readonly bool _optIn, _byPrefix, _optOut; - private readonly byte[]? _cachingYes, _cachingNo; + /// The mode actually in force, and what follows from it; replaced whole, never changed in place. + private ModeState _mode; + + /// + /// The invalidation strategy actually in force: , unless + /// fell back to . + /// + public CacheTrackingMode EffectiveTrackingMode => Mode.Mode; + + /// Whether the mode in force is negotiated with CLIENT TRACKING at the handshake. + internal bool UsesClientTracking => Mode.UsesClientTracking; + + private ModeState Mode => Volatile.Read(ref _mode); + + /// + /// Whether a failure of server-assisted tracking may be answered by falling back, rather than by failing + /// the connection: only when the mode was left to the library. + /// + /// + /// A mode named explicitly is a requirement, and a connection that cannot meet it refuses loudly; see + /// RespHandshake.EnableClientTrackingAsync. is "the library's + /// choice", and the library can choose again. + /// + internal bool CanFallBack => Options.TrackingMode == CacheTrackingMode.Default; + + /// + /// Abandon CLIENT TRACKING for , once and for good. + /// + /// for the one call that made the switch; it owns subscribing and logging. + /// + /// + /// Flushes, because what was filled until now relied on tracking, and nothing will announce those keys + /// until the subscription is live. Entries filled from here on are covered by the subscription, or failing + /// that by , which is what the mode promises either way. + /// + /// + /// Never switches back. Connections that did negotiate tracking keep sending pushes, which only + /// invalidate more than needed; what must stop at once is the CLIENT CACHING frames, which a server + /// that is not tracking rejects - and those are read from the state replaced here. + /// + /// + internal bool TryFallBackToKeyspaceNotifications() + { + if (!CanFallBack) return false; + var current = Mode; + if (!current.UsesClientTracking) return false; + var next = new ModeState(CacheTrackingMode.KeyspaceNotifications, CommandMap.Default); + if (!ReferenceEquals(Interlocked.CompareExchange(ref _mode, next, current), current)) return false; + OnFlush(); + return true; + } + + /// Everything that follows from the mode in force, so that it changes as one. + private sealed class ModeState + { + internal ModeState(CacheTrackingMode mode, CommandMap map) + { + Mode = mode; + OptIn = mode == CacheTrackingMode.TrackingOptIn; + ByPrefix = mode == CacheTrackingMode.TrackingByPrefix; + OptOut = mode == CacheTrackingMode.Tracking; + UsesClientTracking = mode is not (CacheTrackingMode.None or CacheTrackingMode.KeyspaceNotifications); + + // the two CLIENT CACHING frames, rendered once through the command map (CLIENT can be renamed like + // anything else, and the handshake's CLIENT TRACKING honours the same map) and shared by every + // request that needs one: a prefix is a fixed buffer, never pooled, so sharing is free + if (OptIn || ByPrefix) CachingYes = RenderClientCaching(map, "YES"u8); // both negotiate OPTIN + if (OptOut) CachingNo = RenderClientCaching(map, "NO"u8); + } + + internal CacheTrackingMode Mode { get; } + + // OptIn: TrackingOptIn, where only a CacheOptIn read may be kept. ByPrefix: TrackingByPrefix, the same + // wire mode with Prefixes deciding for unflagged reads. OptOut: Tracking, where CacheOptOut has a wire half. + internal bool OptIn { get; } + + internal bool ByPrefix { get; } + + internal bool OptOut { get; } + + internal bool UsesClientTracking { get; } + + internal byte[]? CachingYes { get; } + + internal byte[]? CachingNo { get; } + } /// CLIENT CACHING {verb} as one RESP frame, with the command spelled as the map says. private static byte[] RenderClientCaching(CommandMap map, ReadOnlySpan verb) @@ -174,7 +253,7 @@ private static byte[] RenderClientCaching(CommandMap map, ReadOnlySpan ver /// prefixes - so "about to store" and "needs the prefix" coincide. Elsewhere the server already tracks /// (or broadcasts) everything, and nothing is said. /// - internal byte[]? FillPrefix => _cachingYes; + internal byte[]? FillPrefix => Mode.CachingYes; /// /// The frame to write ahead of a request this cache will not store, or null. @@ -189,7 +268,7 @@ private static byte[] RenderClientCaching(CommandMap map, ReadOnlySpan ver /// for why that one stays local). /// internal byte[]? OptOutPrefix(CommandFlags flags) - => _optOut && (flags & CommandFlags.CacheOptOut) != 0 && IsReadOnly(flags) ? _cachingNo : null; + => Mode is { OptOut: true } mode && (flags & CommandFlags.CacheOptOut) != 0 && IsReadOnly(flags) ? mode.CachingNo : null; /// /// The frame to write ahead of a transaction's MULTI, or null. @@ -204,10 +283,11 @@ private static byte[] RenderClientCaching(CommandMap map, ReadOnlySpan ver /// internal byte[]? TransactionPrefix(List operations) { - if (!_optOut) return null; + var mode = Mode; + if (!mode.OptOut) return null; foreach (var queued in operations) { - if (IsReadOnly(queued.Operation.Flags)) return _cachingNo; + if (IsReadOnly(queued.Operation.Flags)) return mode.CachingNo; } return null; @@ -856,7 +936,8 @@ public bool TryBeginFill(ref RespRequestFrame frame, int database, CommandFlags // ...except under TrackingByPrefix with CacheOptIn said outright: there the list is the policy for // reads that did not state one, the flag is a statement, and the server can track any key we // send behind YES. Under Broadcast the list went to the server, and nothing can cross it. - if (Options.HasPrefixes && !(_byPrefix && (flags & CommandFlags.CacheOptIn) != 0)) + var byPrefix = Mode.ByPrefix; + if (Options.HasPrefixes && !(byPrefix && (flags & CommandFlags.CacheOptIn) != 0)) { for (var i = 0; i < count; i++) { @@ -877,7 +958,7 @@ public bool TryBeginFill(ref RespRequestFrame frame, int database, CommandFlags // opt-in, and a YES that tracks a key we then decline to store would be the worst of both. if (_doorkeeper is not null && (flags & CommandFlags.CacheOptIn) == 0 - && !_byPrefix + && !byPrefix && !_doorkeeper.SeenBefore((frame.AsLookupKey().GetHashCode() * 397) ^ database)) { Interlocked.Increment(ref _refusedNotAdmitted); @@ -1474,7 +1555,7 @@ internal bool PermitsCaching(CommandFlags flags) /// it cannot work for exactly that reason. /// /// - internal bool IsCacheable(CommandFlags flags) => IsCacheable(flags, _optIn); + internal bool IsCacheable(CommandFlags flags) => IsCacheable(flags, Mode.OptIn); /// Whether permit a reply to be served from or stored in a cache. /// The command's flags, with its retry category applied. diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs b/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs index 8c5abe6a6..1341a8035 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.ClientCache.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using StackExchange.Redis.Caching; @@ -6,12 +7,19 @@ namespace StackExchange.Redis; public partial class ConnectionMultiplexer { + private int _cacheSubscribed; + /// - /// Under , subscribe the client-side cache to the keyspace - /// channels that invalidate it; nothing to do in any other mode. + /// When the client-side cache's mode in force is , subscribe it + /// to the keyspace channels that invalidate it; nothing to do otherwise, or if that has already been done. /// /// /// + /// Called after connecting, for a cache configured that way, and by the handshake that makes + /// fall back to it - which can happen at any later reconnect, so this is + /// not only a connect-time step. Once only: a second handler would invalidate every key twice. + /// + /// /// One pattern per prefix (__keyspace@*__:{prefix}*), or a single wildcard without prefixes. Every /// database, because the cache keys its entries by database but invalidates by key alone - as a tracking push /// does - so a notification from another database costs a miss, never a stale read. @@ -29,7 +37,8 @@ public partial class ConnectionMultiplexer /// internal async Task SubscribeCacheInvalidationAsync() { - if (ClientCache is not { Options.ResolvedTrackingMode: CacheTrackingMode.KeyspaceNotifications } cache) return; + if (ClientCache is not { EffectiveTrackingMode: CacheTrackingMode.KeyspaceNotifications } cache) return; + if (Interlocked.Exchange(ref _cacheSubscribed, 1) != 0) return; // the channel is __keyspace@{db}__:{key}, and the key is what the cache matches on, as the bytes it was sent as Action handler = (channel, _) => cache.OnInvalidate(RedisChannel.StripKeySpacePrefix(channel.Span)); diff --git a/src/StackExchange.Redis/Contexts/RespHandshake.cs b/src/StackExchange.Redis/Contexts/RespHandshake.cs index 3f090dccc..5b287e083 100644 --- a/src/StackExchange.Redis/Contexts/RespHandshake.cs +++ b/src/StackExchange.Redis/Contexts/RespHandshake.cs @@ -587,9 +587,10 @@ internal static async Task PerformAsync( // None negotiates nothing, and KeyspaceNotifications subscribes instead (see // ConnectionMultiplexer.SubscribeCacheInvalidationAsync); neither needs RESP3 - if (clientCache is { Options.UsesClientTracking: true }) + // ...and the mode actually in force, not the one configured: Default may already have fallen back + if (clientCache is { UsesClientTracking: true }) { - await EnableClientTrackingAsync(context, clientCache, protocol).ConfigureAwait(false); + await EnableClientTrackingAsync(context, clientCache, protocol, server).ConfigureAwait(false); } if (database > 0) @@ -1316,6 +1317,7 @@ static async Task TellAsync(RespDatabaseContext context, RedisValue verb, RedisV /// The connection being brought up. /// The cache the invalidations are for. /// What the handshake settled on; tracking needs RESP3. + /// The modelled server, for logging and for reaching the multiplexer on a fallback. /// /// /// It has to be THIS connection, which is the whole reason this exists here. Invalidations @@ -1327,14 +1329,29 @@ static async Task TellAsync(RespDatabaseContext context, RedisValue verb, RedisV /// difference made visible. /// /// - /// It refuses loudly rather than degrading, as ServerEndPoint.EnableClientTrackingAsync - /// does: a cache that is filled but never invalidated is silently, durably wrong, so a connection - /// that cannot establish tracking fails instead of serving stale data. + /// A mode named explicitly refuses loudly rather than degrading, as + /// ServerEndPoint.EnableClientTrackingAsync does: a cache that is filled but never invalidated is + /// silently, durably wrong, so a connection that cannot establish tracking fails instead of serving stale + /// data. + /// + /// + /// falls back instead, to + /// : "the library's choice" can be made again, + /// and a cache that connects is what someone who never chose a mode expects. Without RESP3, or when the + /// server answers CLIENT TRACKING with any error - an old server, a proxy, an ACL - the whole cache + /// switches, for good (see ). A + /// connection failure is not a refusal, and still fails the connection. /// /// private static async Task EnableClientTrackingAsync( - RespDatabaseContext context, Caching.RespClientCache cache, RedisProtocol protocol) + RespDatabaseContext context, Caching.RespClientCache cache, RedisProtocol protocol, ServerEndPoint? server) { + if (protocol != RedisProtocol.Resp3 && cache.CanFallBack) + { + await FallBackAsync(context, cache, server, "the connection is not RESP3").ConfigureAwait(false); + return; + } + if (protocol != RedisProtocol.Resp3) { const string Message = @@ -1344,12 +1361,58 @@ private static async Task EnableClientTrackingAsync( throw new RedisConnectionException(ConnectionFailureType.ProtocolFailure, CommandFlags.CommandRetryNever, Message); } + try + { + await SendClientTrackingAsync(context, cache).ConfigureAwait(false); + } + catch (RedisServerException ex) when (cache.CanFallBack) + { + await FallBackAsync(context, cache, server, ex.Message).ConfigureAwait(false); + } + } + + /// Switch the cache to keyspace notifications, and say so. + /// + /// + /// Only the connection that makes the switch subscribes and logs; every later handshake sees a cache that no + /// longer uses tracking and skips all of this. The subscription is started, not awaited: it is made on + /// another connection, which may be waiting for this one, and the flush the switch performs covers the gap. + /// + /// + /// A diagnostic, never a refusal. Keyspace notifications are off by default, and with them off this + /// mode is retired by lifetime alone; when CONFIG GET is permitted it can say so, and that is worth a + /// warning. When it is not - plenty of managed services restrict it - there is nothing to say. + /// + /// + private static async Task FallBackAsync( + RespDatabaseContext context, Caching.RespClientCache cache, ServerEndPoint? server, string reason) + { + if (!cache.TryFallBackToKeyspaceNotifications()) return; + + var multiplexer = server?.Multiplexer; + multiplexer?.Logger?.LogWarningCacheFellBack(new(server!), reason); + multiplexer?.SubscribeCacheInvalidationAsync().ObserveErrors(); + + if (multiplexer?.Logger is { } log && context.Raw.CommandMap.IsAvailable(RedisCommand.CONFIG)) + { + var events = await ReadSettingAsync(context, "notify-keyspace-events").ConfigureAwait(false); + if (events is not null && !Caching.CacheOptions.KeyspaceEventsInvalidate(events)) + { + log.LogWarningCacheKeyspaceEventsDisabled(new(server!), events); + } + } + } + + /// The CLIENT TRACKING command for the mode in force. + private static async Task SendClientTrackingAsync(RespDatabaseContext context, Caching.RespClientCache cache) + { var options = cache.Options; + var mode = cache.EffectiveTrackingMode; var prefixes = options.PrefixBytes; // the snapshot the cache matches on: the same bytes go to the server var command = RedisCommand.CLIENT; // CLIENT TRACKING ON [BCAST] [PREFIX p]... - if (options.ResolvedTrackingMode == Caching.CacheTrackingMode.Broadcast) + if (mode == Caching.CacheTrackingMode.Broadcast) { if (prefixes.Length == 0) { @@ -1382,7 +1445,7 @@ await context.SendAsync($"{command}{RedisLiterals.TRACKING}{RedisLiterals.ON}{Re // read is written behind CLIENT CACHING YES, which is what CommandFlags.CacheOptIn does - and // what TrackingByPrefix does for an unflagged read inside its prefixes. The prefixes themselves // stay client-side there: PREFIX is refused without BCAST. - var refinement = options.ResolvedTrackingMode is Caching.CacheTrackingMode.TrackingOptIn or Caching.CacheTrackingMode.TrackingByPrefix + var refinement = mode is Caching.CacheTrackingMode.TrackingOptIn or Caching.CacheTrackingMode.TrackingByPrefix ? RedisLiterals.OPTIN : RedisLiterals.OPTOUT; await context.SendAsync($"{command}{RedisLiterals.TRACKING}{RedisLiterals.ON}{refinement}").ConfigureAwait(false); diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 31796641c..f6673c32a 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -892,6 +892,18 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "The client-side cache could not subscribe to keyspace notifications; entries will be retired by their time-to-live alone until the subscription is restored: {ErrorMessage}")] internal static partial void LogWarningCacheSubscriptionFailed(this ILogger logger, Exception exception, string errorMessage); + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 139, + Message = "{Server}: server-assisted client-side caching (CLIENT TRACKING) is unavailable ({Reason}); the cache has switched to keyspace notifications")] + internal static partial void LogWarningCacheFellBack(this ILogger logger, ServerEndPointLogValue server, string reason); + + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 140, + Message = "{Server}: notify-keyspace-events is '{Events}', which raises no keyspace events the client-side cache can use; entries will be retired by their time-to-live alone. Include K and the event classes to invalidate on (KA for all).")] + internal static partial void LogWarningCacheKeyspaceEventsDisabled(this ILogger logger, ServerEndPointLogValue server, string events); + [LoggerMessage( Level = LogLevel.Warning, EventId = 130, diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 342dac747..dfb6a78c4 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -81,6 +81,7 @@ StackExchange.Redis.Caching.ClientCache StackExchange.Redis.Caching.ClientCache.Bytes.get -> long StackExchange.Redis.Caching.ClientCache.Count.get -> int StackExchange.Redis.Caching.ClientCache.Evicted.get -> long +StackExchange.Redis.Caching.ClientCache.EffectiveTrackingMode.get -> StackExchange.Redis.Caching.CacheTrackingMode StackExchange.Redis.Caching.ClientCache.Expired.get -> long StackExchange.Redis.Caching.ClientCache.Invalidate(StackExchange.Redis.RedisKey key) -> bool StackExchange.Redis.Caching.ClientCache.Invalidate(System.ReadOnlySpan key) -> bool diff --git a/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs b/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs index d46019dc6..4fc8bcdf5 100644 --- a/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs +++ b/tests/StackExchange.Redis.Tests/RespCacheInvalidationStrategyTests.cs @@ -1,7 +1,9 @@ using System; using System.Diagnostics; +using System.Linq; using System.Threading.Tasks; using StackExchange.Redis.Caching; +using StackExchange.Redis.Server; using Xunit; namespace StackExchange.Redis.Tests; @@ -197,6 +199,122 @@ public async Task KeyspaceNotificationsRefuseAKeyOutsideThePrefixes() Assert.Equal(1, cache.RefusedNotTracked); } + [Theory] + [InlineData("", false)] + [InlineData("K", false)] // the channel, but no events on it + [InlineData("Kn", false)] // new keys have nothing cached + [InlineData("Km", false)] // misses are reads + [InlineData("AE", false)] // keyevent only + [InlineData("KA", true)] + [InlineData("AKE", true)] + [InlineData("K$", true)] + [InlineData("Kgx", true)] + public void KeyspaceEventsThatCanInvalidate(string events, bool expected) + => Assert.Equal(expected, CacheOptions.KeyspaceEventsInvalidate(events)); + + /// A disabled CLIENT is known before connecting: Default starts out on keyspace notifications. + [Fact] + public void DefaultFallsBackAtOnceWhenClientIsDisabled() + { + var map = CommandMap.Create(["CLIENT"], available: false); + using var fallen = new RespClientCache(new CacheOptions(), commandMap: map); + Assert.Equal(CacheTrackingMode.KeyspaceNotifications, fallen.EffectiveTrackingMode); + Assert.False(fallen.UsesClientTracking); + + // a named mode is a requirement, and is not quietly changed + Assert.Throws(() => new RespClientCache(new CacheOptions { TrackingMode = CacheTrackingMode.Tracking }, commandMap: map)); + } + + /// + /// The switch is whole: the CLIENT CACHING frames stop at once - a server that is not tracking rejects + /// them - and the prefixes become the hard boundary they are under keyspace notifications. + /// + [Fact] + public void FallingBackStopsTheClientCachingFramesAndHappensOnce() + { + using var optOut = new RespClientCache(new CacheOptions()); + Assert.Equal(CacheTrackingMode.Tracking, optOut.EffectiveTrackingMode); + Assert.NotNull(optOut.OptOutPrefix(CommandFlags.CacheOptOut | CommandFlags.CommandRetryReadOnly)); + + Assert.True(optOut.TryFallBackToKeyspaceNotifications()); + Assert.False(optOut.TryFallBackToKeyspaceNotifications()); + Assert.Equal(CacheTrackingMode.KeyspaceNotifications, optOut.EffectiveTrackingMode); + Assert.Equal(CacheTrackingMode.KeyspaceNotifications, optOut.View.EffectiveTrackingMode); + Assert.Null(optOut.OptOutPrefix(CommandFlags.CacheOptOut | CommandFlags.CommandRetryReadOnly)); + + using var byPrefix = new RespClientCache(new CacheOptions { Prefixes = ["app:"] }); + Assert.Equal(CacheTrackingMode.TrackingByPrefix, byPrefix.EffectiveTrackingMode); + Assert.NotNull(byPrefix.FillPrefix); + Assert.True(byPrefix.TryFallBackToKeyspaceNotifications()); + Assert.Null(byPrefix.FillPrefix); + + // only Default may fall back + using var named = new RespClientCache(new CacheOptions { TrackingMode = CacheTrackingMode.Tracking }); + Assert.False(named.TryFallBackToKeyspaceNotifications()); + Assert.Equal(CacheTrackingMode.Tracking, named.EffectiveTrackingMode); + } + + /// + /// Default over RESP2 used to refuse to connect; it now connects on keyspace notifications, and they work. + /// + [Fact] + public async Task DefaultOverResp2FallsBackToKeyspaceNotifications() + { + var me = Me() + ":"; + using var muxer = await ConnectAsync(TestConfig.Current.PrimaryServerAndPort, new CacheOptions { Admission = CacheAdmission.OnFirstMiss, Prefixes = [me] }); + Assert.Equal(CacheTrackingMode.KeyspaceNotifications, muxer.GetClientCache()?.EffectiveTrackingMode); + await AssertInvalidatedAsync(muxer, TestConfig.Current.PrimaryServerAndPort, me, keyCount: 1); + } + + /// ...whereas a named tracking mode over RESP2 still refuses, as it always has. + [Fact] + public async Task ANamedTrackingModeOverResp2StillRefuses() + { + var options = ConfigurationOptions.Parse(TestConfig.Current.PrimaryServerAndPort); + options.Protocol = RedisProtocol.Resp2; + options.AbortOnConnectFail = true; + options.ClientCache = new CacheOptions { TrackingMode = CacheTrackingMode.Tracking }; + await Assert.ThrowsAnyAsync(() => ConnectionMultiplexer.ConnectAsync(options, Writer)); + } + + /// + /// A server that refuses CLIENT TRACKING - an old server, a proxy, an ACL - makes Default fall back, and a + /// named mode fail. The fallback is logged, and so - when CONFIG GET can say - are notifications that are + /// off; a diagnostic, never a refusal. + /// + [Theory] + [InlineData("", true)] + [InlineData("K", true)] + [InlineData("KA", false)] + [InlineData(null, false)] // the server does not say: nothing to warn about + public async Task AServerThatRefusesTrackingMakesDefaultFallBack(string? events, bool warnsEventsOff) + { + using var server = new NoTrackingServer(Output, events); + var config = server.GetClientConfig(); + config.Protocol = RedisProtocol.Resp3; + config.ClientCache = new CacheOptions(); + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + using (var muxer = await ConnectionMultiplexer.ConnectAsync(config, Writer)) + { + var cache = muxer.GetClientCache(); + Assert.NotNull(cache); + Assert.Equal(CacheTrackingMode.KeyspaceNotifications, cache.EffectiveTrackingMode); + Assert.True(muxer.IsConnected); + await muxer.GetDatabase().PingAsync(); + } + + Output.WriteLine(logger.All); + Assert.Single(logger.Matching("has switched to keyspace notifications")); + Assert.Equal(warnsEventsOff, logger.Matching("notify-keyspace-events is").Any()); + + config = server.GetClientConfig(); + config.Protocol = RedisProtocol.Resp3; + config.AbortOnConnectFail = true; + config.ClientCache = new CacheOptions { TrackingMode = CacheTrackingMode.Tracking }; + await Assert.ThrowsAnyAsync(() => ConnectionMultiplexer.ConnectAsync(config, Writer)); + } + private async Task AssertInvalidatedAsync(ConnectionMultiplexer muxer, string configuration, string prefix, int keyCount) { var cache = muxer.ClientCache; @@ -227,4 +345,20 @@ await WaitFor(async () => (string?)await db.Strings.GetAsync(key) == "v2"), $"the keyspace notification for '{key}' never invalidated it"); } } + + /// The toy server, answering CLIENT TRACKING as a server without it would. + private sealed class NoTrackingServer : InProcessTestServer + { + public NoTrackingServer(ITestOutputHelper log, string? keyspaceEvents = null) : base(log) + { + // a struct over the server's dictionary: the copy writes through + var configuration = ServerConfiguration; + if (keyspaceEvents is not null) configuration["notify-keyspace-events"] = keyspaceEvents; + } + + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) + => request.Count >= 2 && request.IsString(0, "CLIENT"u8) && request.IsString(1, "TRACKING"u8) + ? TypedRedisValue.Error("ERR unknown subcommand 'TRACKING'") + : base.Execute(client, in request); + } } diff --git a/tests/StackExchange.Redis.Tests/RespInProcTrackingTests.cs b/tests/StackExchange.Redis.Tests/RespInProcTrackingTests.cs index 9d5c4948b..0356d5127 100644 --- a/tests/StackExchange.Redis.Tests/RespInProcTrackingTests.cs +++ b/tests/StackExchange.Redis.Tests/RespInProcTrackingTests.cs @@ -237,19 +237,21 @@ await WaitFor(async () => (string?)await db.Strings.GetAsync("app:k") is null), } /// - /// Without RESP3 the connection fails rather than quietly caching without invalidation. + /// Without RESP3 a named tracking mode fails the connection rather than quietly caching without invalidation. /// /// /// The one case where taking the connection down is the kinder outcome: a cache that fills and is never - /// invalidated is silently and durably wrong, bounded only by the entry lifetime. + /// invalidated is silently and durably wrong, bounded only by the entry lifetime. A mode left at + /// falls back to keyspace notifications instead - see + /// RespCacheInvalidationStrategyTests.DefaultOverResp2FallsBackToKeyspaceNotifications. /// [Fact] - public async Task Resp2WithACacheRefusesLoudly() + public async Task Resp2WithANamedTrackingModeRefusesLoudly() { using var server = new InProcessTestServer(log); var config = server.GetClientConfig(); config.Protocol = RedisProtocol.Resp2; - config.ClientCache = new CacheOptions { Admission = CacheAdmission.OnFirstMiss }; + config.ClientCache = new CacheOptions { Admission = CacheAdmission.OnFirstMiss, TrackingMode = CacheTrackingMode.Tracking }; config.AbortOnConnectFail = true; var connectLog = new StringWriter();