From 9a2fe4cc312cc48f1e13a455ad022f95c443f6a5 Mon Sep 17 00:00:00 2001 From: NSAmelchev Date: Fri, 30 Oct 2026 17:54:28 +0300 Subject: [PATCH 1/2] IGNITE-29233 Fix explicit lock left acquired when its timeout expires during mapping --- .../cache/GridCacheExplicitLockSpan.java | 12 + .../cache/GridCacheMvccCandidate.java | 33 +++ .../dht/colocated/GridDhtColocatedCache.java | 3 +- .../colocated/GridDhtColocatedLockFuture.java | 22 +- .../ExplicitLockTimeoutDuringMappingTest.java | 248 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite14.java | 2 + 6 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java index 8c09abcd5c96c..6ec4115148c2b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java @@ -195,6 +195,18 @@ public GridCacheMvccCandidate removeCandidate(IgniteTxKey key, @Nullable GridCac if (deque.isEmpty()) cands.remove(cand.key()); } + else { + // Another lock of the thread holds the key, the future of this version could add a reentry only. + GridCacheMvccCandidate reentry = first.unenter(ver); + + if (reentry != null) { + boolean rmvd = deque.removeIf(c -> c == reentry); + + assert rmvd : reentry; + + cand = reentry; + } + } } boolean empty = cands.isEmpty(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java index 6810658837104..66943ce7a3c29 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccCandidate.java @@ -70,6 +70,9 @@ public class GridCacheMvccCandidate implements CacheLockCandidates { /** Linked reentry. */ private GridCacheMvccCandidate reentry; + /** Version of the explicit lock future that created this reentry, see {@link #unenter(GridCacheVersion)}. */ + private GridCacheVersion reentryVer; + /** Previous lock for the thread. */ @GridToStringExclude private volatile GridCacheMvccCandidate prev; @@ -198,6 +201,14 @@ public void topologyVersion(AffinityTopologyVersion topVer) { * @return Reentry candidate. */ public GridCacheMvccCandidate reenter() { + return reenter(null); + } + + /** + * @param reentryVer Version of the explicit lock future that creates the reentry, {@code null} otherwise. + * @return Reentry candidate. + */ + public GridCacheMvccCandidate reenter(@Nullable GridCacheVersion reentryVer) { GridCacheMvccCandidate old = reentry; GridCacheMvccCandidate reentry = new GridCacheMvccCandidate( @@ -217,6 +228,7 @@ public GridCacheMvccCandidate reenter() { read()); reentry.topVer = topVer; + reentry.reentryVer = reentryVer; if (old != null) reentry.reentry = old; @@ -242,6 +254,27 @@ public GridCacheMvccCandidate reenter() { return null; } + /** + * Unlinks the reentry created by the explicit lock future of the given version, so that a failed future reverts + * its reentry of a key the thread already holds. + * + * @param reentryVer Version of the future. + * @return Unlinked reentry or {@code null} if the future has not created a reentry of this candidate. + */ + @Nullable public GridCacheMvccCandidate unenter(GridCacheVersion reentryVer) { + for (GridCacheMvccCandidate prev = this; prev.reentry != null; prev = prev.reentry) { + GridCacheMvccCandidate reentry = prev.reentry; + + if (reentryVer.equals(reentry.reentryVer)) { + prev.reentry = reentry.reentry; + + return reentry; + } + } + + return null; + } + /** * @param parent Sets locks parent entry. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java index 5f6d6c13e8302..d9d6c45626ebb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java @@ -824,7 +824,8 @@ public void removeLocks(long threadId, GridCacheVersion ver, Collection keys, AffinityTopologyVersion topVer) + throws IgniteCheckedException { + if (tx != null || timeoutObj == null) + return mapAsPrimary0(keys, topVer); + + // Explicit lock candidates registered after the future has been completed by the timeout or cancellation + // are never released: undoLocks() has already run. The completion holds the same monitor, so the local + // mapping is atomic with it. Transactions release their candidates on rollback instead. + synchronized (timeoutObj) { + return isDone() || mapAsPrimary0(keys, topVer); + } + } + + /** */ + private boolean mapAsPrimary0(Collection keys, AffinityTopologyVersion topVer) throws IgniteCheckedException { // Assign keys to primary nodes. Collection distributedKeys = new ArrayList<>(keys.size()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java new file mode 100644 index 0000000000000..eee912e98cb85 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryReader; +import org.apache.ignite.binary.BinaryWriter; +import org.apache.ignite.binary.Binarylizable; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.internal.processors.cache.GridCacheMvccManager.CACHE_EXPLICIT_LOCKS_VIEW; +import static org.apache.ignite.internal.processors.cache.GridCacheMvccManager.CACHE_LOCKS_VIEW; + +/** + * Explicit lock timeout that expires while the lock is still being mapped, for example after a GC pause longer than + * the timeout or while the mapping waits for an exchange: lock candidates registered by the mapping must not outlive + * the failed lock attempt. + */ +public class ExplicitLockTimeoutDuringMappingTest extends GridCommonAbstractTest { + /** */ + private static final long TIMEOUT = 10_000L; + + /** Pause of the key serialization in the current thread, consumed by the first serialization. */ + private static final ThreadLocal KEY_SERIALIZATION_PAUSE = new ThreadLocal<>(); + + /** */ + private IgniteCache cache; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setCommunicationSpi(new TestRecordingCommunicationSpi()) + .setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME).setAtomicityMode(TRANSACTIONAL)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cache = startGrid(0).cache(DEFAULT_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** The timeout expires while the key is being mapped: the key must not stay locked by nobody. */ + @Test + public void testTimeoutDuringMapping() throws Exception { + StallingKey key = new StallingKey(1); + + Lock lock = cache.lock(key); + + KEY_SERIALIZATION_PAUSE.set(500L); + + if (lock.tryLock(100, TimeUnit.MILLISECONDS)) + lock.unlock(); + + checkReleased(cache, key); + } + + /** The timeout of a reentrant lock expires while the key is being mapped: the reentry must not outlive it. */ + @Test + public void testReentrantLockTimeoutDuringMapping() throws Exception { + StallingKey key = new StallingKey(1); + + Lock outer = cache.lock(key); + + outer.lock(); + + KEY_SERIALIZATION_PAUSE.set(500L); + + Lock inner = cache.lock(key); + + if (inner.tryLock(100, TimeUnit.MILLISECONDS)) + inner.unlock(); + + outer.unlock(); + + checkReleased(cache, key); + } + + /** A multi-key lock with a held key times out on the other key: the reentry of the held key must be reverted. */ + @Test + public void testMultiKeyLockTimeoutWithHeldKey() throws Exception { + Lock outer = cache.lock(1); + + outer.lock(); + + CountDownLatch release = new CountDownLatch(1); + + IgniteInternalFuture holder = holdInOtherThread(cache, 2, release); + + assertFalse(cache.lockAll(Arrays.asList(1, 2)).tryLock(100, TimeUnit.MILLISECONDS)); + + outer.unlock(); + + release.countDown(); + + holder.get(TIMEOUT); + + checkReleased(cache, 1); + } + + /** The timeout expires while the mapping waits for an exchange: the postponed mapping must not lock the key. */ + @Test + public void testTimeoutBeforePostponedMapping() throws Exception { + IgniteEx node = startGrid(1); + + awaitPartitionMapExchange(); + + IgniteCache cache1 = node.cache(DEFAULT_CACHE_NAME); + + Integer key = primaryKey(cache1); + + TestRecordingCommunicationSpi crdSpi = TestRecordingCommunicationSpi.spi(grid(0)); + + crdSpi.blockMessages(GridDhtPartitionsFullMessage.class, node.name()); + + IgniteInternalFuture exchFut = GridTestUtils.runAsync(() -> grid(0).createCache("tmp1")); + + crdSpi.waitForBlocked(); + + assertFalse(cache1.lock(key).tryLock(500, TimeUnit.MILLISECONDS)); + + crdSpi.stopBlock(); + + exchFut.get(TIMEOUT); + + // The next exchange would wait for a leaked candidate forever. + GridTestUtils.runAsync(() -> grid(0).createCache("tmp2")).get(TIMEOUT); + + checkReleased(cache1, key); + } + + /** Checks that the key can be locked by another thread and is not considered locked by the current one. */ + private void checkReleased(IgniteCache cache, Object key) throws Exception { + CountDownLatch release = new CountDownLatch(1); + + IgniteInternalFuture holder = holdInOtherThread(cache, key, release); + + assertFalse("Lock candidate leaked: " + key, cache.lock(key).tryLock()); + + release.countDown(); + + holder.get(TIMEOUT); + + for (Ignite ignite : G.allGrids()) { + assertEquals(0, ((IgniteEx)ignite).context().systemView().view(CACHE_EXPLICIT_LOCKS_VIEW).size()); + assertEquals(0, ((IgniteEx)ignite).context().systemView().view(CACHE_LOCKS_VIEW).size()); + } + } + + /** @return Future of the thread that holds the lock on the key until the latch is released. */ + private IgniteInternalFuture holdInOtherThread( + IgniteCache cache, + Object key, + CountDownLatch release + ) throws Exception { + CountDownLatch locked = new CountDownLatch(1); + + IgniteInternalFuture holder = GridTestUtils.runAsync(() -> { + Lock lock = cache.lock(key); + + lock.lock(); + + locked.countDown(); + + release.await(); + + lock.unlock(); + + return null; + }); + + assertTrue("Key stays locked: " + key, locked.await(TIMEOUT, TimeUnit.MILLISECONDS)); + + return holder; + } + + /** Key whose serialization pauses the thread, see {@link #KEY_SERIALIZATION_PAUSE}. */ + private static class StallingKey implements Binarylizable { + /** */ + private int id; + + /** */ + private StallingKey(int id) { + this.id = id; + } + + /** {@inheritDoc} */ + @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException { + Long pause = KEY_SERIALIZATION_PAUSE.get(); + + if (pause != null) { + KEY_SERIALIZATION_PAUSE.remove(); + + doSleep(pause); + } + + writer.writeInt("id", id); + } + + /** {@inheritDoc} */ + @Override public void readBinary(BinaryReader reader) throws BinaryObjectException { + id = reader.readInt("id"); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "StallingKey [id=" + id + ']'; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java index b48b4ef05953e..12f00682c021f 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java @@ -51,6 +51,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.CachePartitionPartialCountersMapSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.ExplicitLockCancelOnNodeStopTest; import org.apache.ignite.internal.processors.cache.distributed.dht.ExplicitLockDumpTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.ExplicitLockTimeoutDuringMappingTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedDebugTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPreloadRestartSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPrimarySyncSelfTest; @@ -131,6 +132,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, ExplicitLockCancelOnNodeStopTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, ExplicitLockDumpTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, ExplicitLockKillTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, ExplicitLockTimeoutDuringMappingTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNestedTxTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxConcurrentGetTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxReadTest.class, ignoredTests); From d578eb20eacbe72f18363ce7562a2a3cd5c64d33 Mon Sep 17 00:00:00 2001 From: NSAmelchev Date: Fri, 30 Oct 2026 23:29:49 +0300 Subject: [PATCH 2/2] IGNITE-29233 Fix explicit lock left acquired when its timeout expires during mapping --- .../colocated/GridDhtColocatedLockFuture.java | 31 +++---- .../ExplicitLockTimeoutDuringMappingTest.java | 91 ++++++++++++++++++- 2 files changed, 100 insertions(+), 22 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java index d6eade896d351..9508efd499124 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java @@ -602,7 +602,7 @@ private synchronized void onError(Throwable t) { } } else if (timeoutObj != null) { - // Serialized with the registration of explicit lock candidates, see mapAsPrimary(). + // Serialized with the registration of explicit lock candidates, see map(). synchronized (timeoutObj) { return onComplete(false, true); } @@ -939,10 +939,17 @@ private void mapOnTopology(final boolean remap, @Nullable final Runnable c) { */ private void map(Collection keys, boolean remap, boolean topLocked) { try { - map0( - keys, - remap, - topLocked); + if (tx != null || timeoutObj == null) + map0(keys, remap, topLocked); + else { + // Candidates registered after the future has been completed by the timeout or cancellation are never + // released: undoLocks() has already run. The completion holds the same monitor, so the mapping is + // atomic with it. Transactions release their candidates on rollback instead. + synchronized (timeoutObj) { + if (!isDone()) + map0(keys, remap, topLocked); + } + } } catch (IgniteCheckedException ex) { onDone(false, ex); @@ -1357,20 +1364,6 @@ private void markExplicitOwner(KeyCacheObject key) { * @throws IgniteCheckedException If key cannot be added to mapping. */ private boolean mapAsPrimary(Collection keys, AffinityTopologyVersion topVer) - throws IgniteCheckedException { - if (tx != null || timeoutObj == null) - return mapAsPrimary0(keys, topVer); - - // Explicit lock candidates registered after the future has been completed by the timeout or cancellation - // are never released: undoLocks() has already run. The completion holds the same monitor, so the local - // mapping is atomic with it. Transactions release their candidates on rollback instead. - synchronized (timeoutObj) { - return isDone() || mapAsPrimary0(keys, topVer); - } - } - - /** */ - private boolean mapAsPrimary0(Collection keys, AffinityTopologyVersion topVer) throws IgniteCheckedException { // Assign keys to primary nodes. Collection distributedKeys = new ArrayList<>(keys.size()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java index eee912e98cb85..e22b675972f6a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockTimeoutDuringMappingTest.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -29,9 +30,12 @@ import org.apache.ignite.binary.Binarylizable; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCacheExplicitLockSpan; +import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.testframework.GridTestUtils; @@ -51,9 +55,15 @@ public class ExplicitLockTimeoutDuringMappingTest extends GridCommonAbstractTest /** */ private static final long TIMEOUT = 10_000L; - /** Pause of the key serialization in the current thread, consumed by the first serialization. */ + /** + * Pause of the first serialization of a key in the current thread. The keys are converted lazily by the mapping, + * so the pause lands after the lock timeout is armed. + */ private static final ThreadLocal KEY_SERIALIZATION_PAUSE = new ThreadLocal<>(); + /** */ + private static final AtomicBoolean STALL_EXPLICIT_LOCK_REGISTRATION = new AtomicBoolean(); + /** */ private IgniteCache cache; @@ -167,6 +177,48 @@ public void testTimeoutBeforePostponedMapping() throws Exception { checkReleased(cache1, key); } + /** + * The timeout expires while the candidate of a key of a remote primary node is registered on a client node: the + * candidate must not outlive the failed lock attempt. + */ + @Test + public void testTimeoutDuringRemoteMappingOnClient() throws Exception { + IgniteEx client = startClientGrid(1); + + awaitPartitionMapExchange(); + + checkTimeoutDuringRemoteMapping(client.cache(DEFAULT_CACHE_NAME), 1); + } + + /** + * The timeout expires while the candidate of a key of a remote primary node is registered on a server node: the + * candidate must not outlive the failed lock attempt. + */ + @Test + public void testTimeoutDuringRemoteMappingOnServer() throws Exception { + IgniteEx node = startGrid(1); + + awaitPartitionMapExchange(); + + checkTimeoutDuringRemoteMapping(cache, primaryKey(node.cache(DEFAULT_CACHE_NAME))); + } + + /** */ + private void checkTimeoutDuringRemoteMapping(IgniteCache cache, int key) throws Exception { + STALL_EXPLICIT_LOCK_REGISTRATION.set(true); + + Lock lock = cache.lock(new StallingCacheKey(key)); + + boolean locked = lock.tryLock(100, TimeUnit.MILLISECONDS); + + assertFalse("Registration was not stalled", STALL_EXPLICIT_LOCK_REGISTRATION.get()); + + if (locked) + lock.unlock(); + + checkReleased(cache, key); + } + /** Checks that the key can be locked by another thread and is not considered locked by the current one. */ private void checkReleased(IgniteCache cache, Object key) throws Exception { CountDownLatch release = new CountDownLatch(1); @@ -180,8 +232,13 @@ private void checkReleased(IgniteCache cache, Object key) throws holder.get(TIMEOUT); for (Ignite ignite : G.allGrids()) { - assertEquals(0, ((IgniteEx)ignite).context().systemView().view(CACHE_EXPLICIT_LOCKS_VIEW).size()); - assertEquals(0, ((IgniteEx)ignite).context().systemView().view(CACHE_LOCKS_VIEW).size()); + GridKernalContext ctx = ((IgniteEx)ignite).context(); + + assertEquals(0, ctx.systemView().view(CACHE_EXPLICIT_LOCKS_VIEW).size()); + + // A lock on a remote primary node is removed asynchronously. + assertTrue(GridTestUtils.waitForCondition(() -> ctx.systemView().view(CACHE_LOCKS_VIEW).size() == 0, + TIMEOUT)); } } @@ -245,4 +302,32 @@ private StallingKey(int id) { return "StallingKey [id=" + id + ']'; } } + + /** + * Key pausing the thread inside the registration of its explicit lock candidate: the first access to the key by + * the explicit lock span of the thread adds the candidate, see {@link #STALL_EXPLICIT_LOCK_REGISTRATION}. + */ + private static class StallingCacheKey extends KeyCacheObjectImpl { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private StallingCacheKey(int val) { + super(val, null, -1); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + if (inExplicitLockSpan() && STALL_EXPLICIT_LOCK_REGISTRATION.compareAndSet(true, false)) + doSleep(500); + + return super.hashCode(); + } + + /** */ + private static boolean inExplicitLockSpan() { + return StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE) + .walk(frames -> frames.anyMatch(f -> f.getDeclaringClass() == GridCacheExplicitLockSpan.class)); + } + } }