e : enabledCacheEngine.getMapForTests()
.entrySet()) {
candidatesFound++;
HFileBlock block = (HFileBlock) e.getValue().getBuffer();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
deleted file mode 100644
index e2248cb9e6fa..000000000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ /dev/null
@@ -1,998 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.io.hfile;
-
-import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
-import org.apache.hadoop.hbase.io.HeapSize;
-import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread;
-import org.apache.hadoop.hbase.nio.ByteBuff;
-import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.ClassSize;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tests the concurrent LruBlockCache.
- *
- * Tests will ensure it grows and shrinks in size properly, evictions run when they're supposed to
- * and do what they should, and that cached blocks are accessible when expected to be.
- */
-@Tag(IOTests.TAG)
-@Tag(SmallTests.TAG)
-public class TestLruBlockCache {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestLruBlockCache.class);
-
- private static final Configuration CONF = HBaseConfiguration.create();
-
- @Test
- public void testCacheEvictionThreadSafe() throws Exception {
- long maxSize = 100000;
- int numBlocks = 9;
- int testRuns = 10;
- final long blockSize = calculateBlockSizeDefault(maxSize, numBlocks);
- assertTrue(blockSize * numBlocks <= maxSize, "calculateBlockSize appears broken.");
-
- final LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
- EvictionThread evictionThread = cache.getEvictionThread();
- assertTrue(evictionThread != null);
- Waiter.waitFor(CONF, 10000, 100, () -> evictionThread.isEnteringRun());
- final String hfileName = "hfile";
- int threads = 10;
- final int blocksPerThread = 5 * numBlocks;
- for (int run = 0; run != testRuns; ++run) {
- final AtomicInteger blockCount = new AtomicInteger(0);
- ExecutorService service = Executors.newFixedThreadPool(threads);
- for (int i = 0; i != threads; ++i) {
- service.execute(new Runnable() {
- @Override
- public void run() {
- for (int blockIndex = 0; blockIndex < blocksPerThread
- || (!cache.isEvictionInProgress()); ++blockIndex) {
- CachedItem block =
- new CachedItem(hfileName, (int) blockSize, blockCount.getAndIncrement());
- boolean inMemory = Math.random() > 0.5;
- cache.cacheBlock(block.cacheKey, block, inMemory);
- }
- cache.evictBlocksByHfileName(hfileName);
- }
- });
- }
- service.shutdown();
- // The test may fail here if the evict thread frees the blocks too fast
- service.awaitTermination(10, TimeUnit.MINUTES);
- Waiter.waitFor(CONF, 10000, 100, new ExplainingPredicate() {
- @Override
- public boolean evaluate() throws Exception {
- return cache.getBlockCount() == 0;
- }
-
- @Override
- public String explainFailure() throws Exception {
- return "Cache block count failed to return to 0";
- }
- });
- assertEquals(0, cache.getBlockCount());
- assertEquals(cache.getOverhead(), cache.getCurrentSize());
- }
- }
-
- @Test
- public void testBackgroundEvictionThread() throws Exception {
- long maxSize = 100000;
- int numBlocks = 9;
- long blockSize = calculateBlockSizeDefault(maxSize, numBlocks);
- assertTrue(blockSize * numBlocks <= maxSize, "calculateBlockSize appears broken.");
-
- LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
- EvictionThread evictionThread = cache.getEvictionThread();
- assertTrue(evictionThread != null);
-
- CachedItem[] blocks = generateFixedBlocks(numBlocks + 1, blockSize, "block");
-
- // Make sure eviction thread has entered run method
- Waiter.waitFor(CONF, 10000, 10, () -> evictionThread.isEnteringRun());
-
- // Add all the blocks
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- }
-
- // wait until at least one eviction has run
- Waiter.waitFor(CONF, 30000, 200, new ExplainingPredicate() {
-
- @Override
- public boolean evaluate() throws Exception {
- return cache.getStats().getEvictionCount() > 0;
- }
-
- @Override
- public String explainFailure() throws Exception {
- return "Eviction never happened.";
- }
- });
-
- // let cache stabilize
- // On some systems, the cache will run multiple evictions before it attains
- // steady-state. For instance, after populating the cache with 10 blocks,
- // the first eviction evicts a single block and then a second eviction
- // evicts another. I think this is due to the delta between minSize and
- // acceptableSize, combined with variance between object overhead on
- // different environments.
- int n = 0;
- for (long prevCnt = 0 /* < number of blocks added */, curCnt = cache.getBlockCount(); prevCnt
- != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) {
- Thread.sleep(200);
- assertTrue(n++ < 100, "Cache never stabilized.");
- }
-
- long evictionCount = cache.getStats().getEvictionCount();
- assertTrue(evictionCount >= 1);
- LOG.info("Background Evictions run: {}", evictionCount);
- }
-
- @Test
- public void testCacheSimple() throws Exception {
- long maxSize = 1000000;
- long blockSize = calculateBlockSizeDefault(maxSize, 101);
-
- LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
-
- CachedItem[] blocks = generateRandomBlocks(100, blockSize);
-
- long expectedCacheSize = cache.heapSize();
-
- // Confirm empty
- for (CachedItem block : blocks) {
- assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null);
- }
-
- // Add blocks
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- expectedCacheSize += block.cacheBlockHeapSize();
- }
-
- // Verify correctly calculated cache heap size
- assertEquals(expectedCacheSize, cache.heapSize());
-
- // Check if all blocks are properly cached and retrieved
- for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
- assertTrue(buf != null);
- assertEquals(buf.heapSize(), block.heapSize());
- }
-
- // Re-add same blocks and ensure nothing has changed
- long expectedBlockCount = cache.getBlockCount();
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- }
- assertEquals(expectedBlockCount, cache.getBlockCount(),
- "Cache should ignore cache requests for blocks already in cache");
-
- // Verify correctly calculated cache heap size
- assertEquals(expectedCacheSize, cache.heapSize());
-
- // Check if all blocks are properly cached and retrieved
- for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
- assertTrue(buf != null);
- assertEquals(buf.heapSize(), block.heapSize());
- }
-
- CacheTestUtils.testConvertToJSON(cache);
-
- // Expect no evictions
- assertEquals(0, cache.getStats().getEvictionCount());
- Thread t = new LruBlockCache.StatisticsThread(cache);
- t.start();
- t.join();
- }
-
- @Test
- public void testCacheEvictionSimple() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSizeDefault(maxSize, 10);
-
- LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false);
-
- CachedItem[] blocks = generateFixedBlocks(10, blockSize, "block");
-
- long expectedCacheSize = cache.heapSize();
-
- // Add all the blocks
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- expectedCacheSize += block.cacheBlockHeapSize();
- }
-
- // A single eviction run should have occurred
- assertEquals(1, cache.getStats().getEvictionCount());
-
- // Our expected size overruns acceptable limit
- assertTrue(expectedCacheSize > (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
-
- // But the cache did not grow beyond max
- assertTrue(cache.heapSize() < maxSize);
-
- // And is still below the acceptable limit
- assertTrue(cache.heapSize() < (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
-
- // All blocks except block 0 should be in the cache
- assertTrue(cache.getBlock(blocks[0].cacheKey, true, false, true) == null);
- for (int i = 1; i < blocks.length; i++) {
- assertEquals(cache.getBlock(blocks[i].cacheKey, true, false, true), blocks[i]);
- }
- }
-
- @Test
- public void testCacheEvictionTwoPriorities() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSizeDefault(maxSize, 10);
-
- LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false);
-
- CachedItem[] singleBlocks = generateFixedBlocks(5, 10000, "single");
- CachedItem[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
-
- long expectedCacheSize = cache.heapSize();
-
- // Add and get the multi blocks
- for (CachedItem block : multiBlocks) {
- cache.cacheBlock(block.cacheKey, block);
- expectedCacheSize += block.cacheBlockHeapSize();
- assertEquals(cache.getBlock(block.cacheKey, true, false, true), block);
- }
-
- // Add the single blocks (no get)
- for (CachedItem block : singleBlocks) {
- cache.cacheBlock(block.cacheKey, block);
- expectedCacheSize += block.heapSize();
- }
-
- // A single eviction run should have occurred
- assertEquals(1, cache.getStats().getEvictionCount());
-
- // We expect two entries evicted
- assertEquals(2, cache.getStats().getEvictedCount());
-
- // Our expected size overruns acceptable limit
- assertTrue(expectedCacheSize > (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
-
- // But the cache did not grow beyond max
- assertTrue(cache.heapSize() <= maxSize);
-
- // And is now below the acceptable limit
- assertTrue(cache.heapSize() <= (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
-
- // We expect fairness across the two priorities.
- // This test makes multi go barely over its limit, in-memory
- // empty, and the rest in single. Two single evictions and
- // one multi eviction expected.
- assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false, true) == null);
- assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false, true) == null);
-
- // And all others to be cached
- for (int i = 1; i < 4; i++) {
- assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false, true), singleBlocks[i]);
- assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false, true), multiBlocks[i]);
- }
- }
-
- @Test
- public void testCacheEvictionThreePriorities() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.98f, // min
- 0.99f, // acceptable
- 0.33f, // single
- 0.33f, // multi
- 0.34f, // memory
- 1.2f, // limit
- false, 16 * 1024 * 1024);
-
- CachedItem[] singleBlocks = generateFixedBlocks(5, blockSize, "single");
- CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
- CachedItem[] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
-
- long expectedCacheSize = cache.heapSize();
-
- // Add 3 blocks from each priority
- for (int i = 0; i < 3; i++) {
-
- // Just add single blocks
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
- expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
-
- // Add and get multi blocks
- cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
- expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
- cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
-
- // Add memory blocks as such
- cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
- expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize();
-
- }
-
- // Do not expect any evictions yet
- assertEquals(0, cache.getStats().getEvictionCount());
-
- // Verify cache size
- assertEquals(expectedCacheSize, cache.heapSize());
-
- // Insert a single block, oldest single should be evicted
- cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]);
-
- // Single eviction, one thing evicted
- assertEquals(1, cache.getStats().getEvictionCount());
- assertEquals(1, cache.getStats().getEvictedCount());
-
- // Verify oldest single block is the one evicted
- assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
-
- // Change the oldest remaining single block to a multi
- cache.getBlock(singleBlocks[1].cacheKey, true, false, true);
-
- // Insert another single block
- cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
-
- // Two evictions, two evicted.
- assertEquals(2, cache.getStats().getEvictionCount());
- assertEquals(2, cache.getStats().getEvictedCount());
-
- // Oldest multi block should be evicted now
- assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
-
- // Insert another memory block
- cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
-
- // Three evictions, three evicted.
- assertEquals(3, cache.getStats().getEvictionCount());
- assertEquals(3, cache.getStats().getEvictedCount());
-
- // Oldest memory block should be evicted now
- assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true));
-
- // Add a block that is twice as big (should force two evictions)
- CachedItem[] bigBlocks = generateFixedBlocks(3, blockSize * 3, "big");
- cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]);
-
- // Four evictions, six evicted (inserted block 3X size, expect +3 evicted)
- assertEquals(4, cache.getStats().getEvictionCount());
- assertEquals(6, cache.getStats().getEvictedCount());
-
- // Expect three remaining singles to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true));
-
- // Make the big block a multi block
- cache.getBlock(bigBlocks[0].cacheKey, true, false, true);
-
- // Cache another single big block
- cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
-
- // Five evictions, nine evicted (3 new)
- assertEquals(5, cache.getStats().getEvictionCount());
- assertEquals(9, cache.getStats().getEvictedCount());
-
- // Expect three remaining multis to be evicted
- assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true));
-
- // Cache a big memory block
- cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
-
- // Six evictions, twelve evicted (3 new)
- assertEquals(6, cache.getStats().getEvictionCount());
- assertEquals(12, cache.getStats().getEvictedCount());
-
- // Expect three remaining in-memory to be evicted
- assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false, true));
- }
-
- @Test
- public void testCacheEvictionInMemoryForceMode() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.98f, // min
- 0.99f, // acceptable
- 0.2f, // single
- 0.3f, // multi
- 0.5f, // memory
- 1.2f, // limit
- true, 16 * 1024 * 1024);
-
- CachedItem[] singleBlocks = generateFixedBlocks(10, blockSize, "single");
- CachedItem[] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
- CachedItem[] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
-
- long expectedCacheSize = cache.heapSize();
-
- // 0. Add 5 single blocks and 4 multi blocks to make cache full, si:mu:me = 5:4:0
- for (int i = 0; i < 4; i++) {
- // Just add single blocks
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
- expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
- // Add and get multi blocks
- cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
- expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
- cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
- }
- // 5th single block
- cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
- expectedCacheSize += singleBlocks[4].cacheBlockHeapSize();
- // Do not expect any evictions yet
- assertEquals(0, cache.getStats().getEvictionCount());
- // Verify cache size
- assertEquals(expectedCacheSize, cache.heapSize());
-
- // 1. Insert a memory block, oldest single should be evicted, si:mu:me = 4:4:1
- cache.cacheBlock(memoryBlocks[0].cacheKey, memoryBlocks[0], true);
- // Single eviction, one block evicted
- assertEquals(1, cache.getStats().getEvictionCount());
- assertEquals(1, cache.getStats().getEvictedCount());
- // Verify oldest single block (index = 0) is the one evicted
- assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
-
- // 2. Insert another memory block, another single evicted, si:mu:me = 3:4:2
- cache.cacheBlock(memoryBlocks[1].cacheKey, memoryBlocks[1], true);
- // Two evictions, two evicted.
- assertEquals(2, cache.getStats().getEvictionCount());
- assertEquals(2, cache.getStats().getEvictedCount());
- // Current oldest single block (index = 1) should be evicted now
- assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
-
- // 3. Insert 4 memory blocks, 2 single and 2 multi evicted, si:mu:me = 1:2:6
- cache.cacheBlock(memoryBlocks[2].cacheKey, memoryBlocks[2], true);
- cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
- cache.cacheBlock(memoryBlocks[4].cacheKey, memoryBlocks[4], true);
- cache.cacheBlock(memoryBlocks[5].cacheKey, memoryBlocks[5], true);
- // Three evictions, three evicted.
- assertEquals(6, cache.getStats().getEvictionCount());
- assertEquals(6, cache.getStats().getEvictedCount());
- // two oldest single blocks and two oldest multi blocks evicted
- assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
-
- // 4. Insert 3 memory blocks, the remaining 1 single and 2 multi evicted
- // si:mu:me = 0:0:9
- cache.cacheBlock(memoryBlocks[6].cacheKey, memoryBlocks[6], true);
- cache.cacheBlock(memoryBlocks[7].cacheKey, memoryBlocks[7], true);
- cache.cacheBlock(memoryBlocks[8].cacheKey, memoryBlocks[8], true);
- // Three evictions, three evicted.
- assertEquals(9, cache.getStats().getEvictionCount());
- assertEquals(9, cache.getStats().getEvictedCount());
- // one oldest single block and two oldest multi blocks evicted
- assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[3].cacheKey, true, false, true));
-
- // 5. Insert one memory block, the oldest memory evicted
- // si:mu:me = 0:0:9
- cache.cacheBlock(memoryBlocks[9].cacheKey, memoryBlocks[9], true);
- // one eviction, one evicted.
- assertEquals(10, cache.getStats().getEvictionCount());
- assertEquals(10, cache.getStats().getEvictedCount());
- // oldest memory block evicted
- assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true));
-
- // 6. Insert one new single block, itself evicted immediately since
- // all blocks in cache are memory-type which have higher priority
- // si:mu:me = 0:0:9 (no change)
- cache.cacheBlock(singleBlocks[9].cacheKey, singleBlocks[9]);
- // one eviction, one evicted.
- assertEquals(11, cache.getStats().getEvictionCount());
- assertEquals(11, cache.getStats().getEvictedCount());
- // the single block just cached now evicted (can't evict memory)
- assertEquals(null, cache.getBlock(singleBlocks[9].cacheKey, true, false, true));
- }
-
- // test scan resistance
- @Test
- public void testScanResistance() throws Exception {
-
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
- 0.99f, // acceptable
- 0.33f, // single
- 0.33f, // multi
- 0.34f, // memory
- 1.2f, // limit
- false, 16 * 1024 * 1024);
-
- CachedItem[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
- CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
-
- // Add 5 multi blocks
- for (CachedItem block : multiBlocks) {
- cache.cacheBlock(block.cacheKey, block);
- cache.getBlock(block.cacheKey, true, false, true);
- }
-
- // Add 5 single blocks
- for (int i = 0; i < 5; i++) {
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
- }
-
- // An eviction ran
- assertEquals(1, cache.getStats().getEvictionCount());
-
- // To drop down to 2/3 capacity, we'll need to evict 4 blocks
- assertEquals(4, cache.getStats().getEvictedCount());
-
- // Should have been taken off equally from single and multi
- assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
-
- // Let's keep "scanning" by adding single blocks. From here on we only
- // expect evictions from the single bucket.
-
- // Every time we reach 10 total blocks (every 4 inserts) we get 4 single
- // blocks evicted. Inserting 13 blocks should yield 3 more evictions and
- // 12 more evicted.
-
- for (int i = 5; i < 18; i++) {
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
- }
-
- // 4 total evictions, 16 total evicted
- assertEquals(4, cache.getStats().getEvictionCount());
- assertEquals(16, cache.getStats().getEvictedCount());
-
- // Should now have 7 total blocks
- assertEquals(7, cache.getBlockCount());
-
- }
-
- @Test
- public void testMaxBlockSize() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
- 0.99f, // acceptable
- 0.33f, // single
- 0.33f, // multi
- 0.34f, // memory
- 1.2f, // limit
- false, 1024);
- CachedItem[] tooLong = generateFixedBlocks(10, 1024 + 5, "long");
- CachedItem[] small = generateFixedBlocks(15, 600, "small");
-
- for (CachedItem i : tooLong) {
- cache.cacheBlock(i.cacheKey, i);
- }
- for (CachedItem i : small) {
- cache.cacheBlock(i.cacheKey, i);
- }
- assertEquals(15, cache.getBlockCount());
- for (CachedItem i : small) {
- assertNotNull(cache.getBlock(i.cacheKey, true, false, false));
- }
- for (CachedItem i : tooLong) {
- assertNull(cache.getBlock(i.cacheKey, true, false, false));
- }
-
- assertEquals(10, cache.getStats().getFailedInserts());
- }
-
- // test setMaxSize
- @Test
- public void testResizeBlockCache() throws Exception {
- long maxSize = 300000;
- long blockSize = calculateBlockSize(maxSize, 31);
-
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.98f, // min
- 0.99f, // acceptable
- 0.33f, // single
- 0.33f, // multi
- 0.34f, // memory
- 1.2f, // limit
- false, 16 * 1024 * 1024);
-
- CachedItem[] singleBlocks = generateFixedBlocks(10, blockSize, "single");
- CachedItem[] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
- CachedItem[] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
-
- // Add all blocks from all priorities
- for (int i = 0; i < 10; i++) {
- // Just add single blocks
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
-
- // Add and get multi blocks
- cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
- cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
-
- // Add memory blocks as such
- cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
- }
-
- // Do not expect any evictions yet
- assertEquals(0, cache.getStats().getEvictionCount());
-
- // Resize to half capacity plus an extra block (otherwise we evict an extra)
- cache.setMaxSize((long) (maxSize * 0.5f));
-
- // Should have run a single eviction
- assertEquals(1, cache.getStats().getEvictionCount());
-
- // And we expect 1/2 of the blocks to be evicted
- assertEquals(15, cache.getStats().getEvictedCount());
-
- // And the oldest 5 blocks from each category should be gone
- for (int i = 0; i < 5; i++) {
- assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false, true));
- assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
- }
-
- // And the newest 5 blocks should still be accessible
- for (int i = 5; i < 10; i++) {
- assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false, true));
- assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false, true));
- assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
- }
- }
-
- // test metricsPastNPeriods
- @Test
- public void testPastNPeriodsMetrics() throws Exception {
- double delta = 0.01;
-
- // 3 total periods
- CacheStats stats = new CacheStats("test", 3);
-
- // No accesses, should be 0
- stats.rollMetricsPeriod();
- assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 1, 1 hit caching, 1 hit non-caching, 2 miss non-caching
- // should be (2/4)=0.5 and (1/1)=1
- stats.hit(false, true, BlockType.DATA);
- stats.hit(true, true, BlockType.DATA);
- stats.miss(false, false, BlockType.DATA);
- stats.miss(false, false, BlockType.DATA);
- stats.rollMetricsPeriod();
- assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 2, 1 miss caching, 3 miss non-caching
- // should be (2/8)=0.25 and (1/2)=0.5
- stats.miss(true, false, BlockType.DATA);
- stats.miss(false, false, BlockType.DATA);
- stats.miss(false, false, BlockType.DATA);
- stats.miss(false, false, BlockType.DATA);
- stats.rollMetricsPeriod();
- assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 3, 2 hits of each type
- // should be (6/12)=0.5 and (3/4)=0.75
- stats.hit(false, true, BlockType.DATA);
- stats.hit(true, true, BlockType.DATA);
- stats.hit(false, true, BlockType.DATA);
- stats.hit(true, true, BlockType.DATA);
- stats.rollMetricsPeriod();
- assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 4, evict period 1, two caching misses
- // should be (4/10)=0.4 and (2/5)=0.4
- stats.miss(true, false, BlockType.DATA);
- stats.miss(true, false, BlockType.DATA);
- stats.rollMetricsPeriod();
- assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 5, evict period 2, 2 caching misses, 2 non-caching hit
- // should be (6/10)=0.6 and (2/6)=1/3
- stats.miss(true, false, BlockType.DATA);
- stats.miss(true, false, BlockType.DATA);
- stats.hit(false, true, BlockType.DATA);
- stats.hit(false, true, BlockType.DATA);
- stats.rollMetricsPeriod();
- assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta);
- assertEquals((double) 1 / 3, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 6, evict period 3
- // should be (2/6)=1/3 and (0/4)=0
- stats.rollMetricsPeriod();
- assertEquals((double) 1 / 3, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 7, evict period 4
- // should be (2/4)=0.5 and (0/2)=0
- stats.rollMetricsPeriod();
- assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 8, evict period 5
- // should be 0 and 0
- stats.rollMetricsPeriod();
- assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
-
- // period 9, one of each
- // should be (2/4)=0.5 and (1/2)=0.5
- stats.miss(true, false, BlockType.DATA);
- stats.miss(false, false, BlockType.DATA);
- stats.hit(true, true, BlockType.DATA);
- stats.hit(false, true, BlockType.DATA);
- stats.rollMetricsPeriod();
- assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
- assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
- }
-
- @Test
- public void testCacheBlockNextBlockMetadataMissing() {
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
- int size = 100;
- int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
- byte[] byteArr = new byte[length];
- ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
- HFileContext meta = new HFileContextBuilder().build();
- HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1,
- ByteBuff.wrap(buf), HFileBlock.FILL_HEADER, -1, 52, -1, meta, HEAP);
- HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1,
- ByteBuff.wrap(buf), HFileBlock.FILL_HEADER, -1, -1, -1, meta, HEAP);
-
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
- 0.99f, // acceptable
- 0.33f, // single
- 0.33f, // multi
- 0.34f, // memory
- 1.2f, // limit
- false, 1024);
-
- BlockCacheKey key = new BlockCacheKey("key1", 0);
- ByteBuffer actualBuffer = ByteBuffer.allocate(length);
- ByteBuffer block1Buffer = ByteBuffer.allocate(length);
- ByteBuffer block2Buffer = ByteBuffer.allocate(length);
- blockWithNextBlockMetadata.serialize(block1Buffer, true);
- blockWithoutNextBlockMetadata.serialize(block2Buffer, true);
-
- // Add blockWithNextBlockMetadata, expect blockWithNextBlockMetadata back.
- CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata, actualBuffer,
- block1Buffer);
-
- // Add blockWithoutNextBlockMetada, expect blockWithNextBlockMetadata back.
- CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer,
- block1Buffer);
-
- // Clear and add blockWithoutNextBlockMetadata
- cache.clearCache();
- assertNull(cache.getBlock(key, false, false, false));
- CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer,
- block2Buffer);
-
- // Add blockWithNextBlockMetadata, expect blockWithNextBlockMetadata to replace.
- CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata, actualBuffer,
- block1Buffer);
- }
-
- private CachedItem[] generateFixedBlocks(int numBlocks, int size, String pfx) {
- CachedItem[] blocks = new CachedItem[numBlocks];
- for (int i = 0; i < numBlocks; i++) {
- blocks[i] = new CachedItem(pfx + i, size);
- }
- return blocks;
- }
-
- private CachedItem[] generateFixedBlocks(int numBlocks, long size, String pfx) {
- return generateFixedBlocks(numBlocks, (int) size, pfx);
- }
-
- private CachedItem[] generateRandomBlocks(int numBlocks, long maxSize) {
- CachedItem[] blocks = new CachedItem[numBlocks];
- Random rand = ThreadLocalRandom.current();
- for (int i = 0; i < numBlocks; i++) {
- blocks[i] = new CachedItem("block" + i, rand.nextInt((int) maxSize) + 1);
- }
- return blocks;
- }
-
- private long calculateBlockSize(long maxSize, int numBlocks) {
- long roughBlockSize = maxSize / numBlocks;
- int numEntries = (int) Math.ceil((1.2) * maxSize / roughBlockSize);
- long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP
- + (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY)
- + (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
- long negateBlockSize = (long) (totalOverhead / numEntries);
- negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
- return ClassSize.align((long) Math.floor((roughBlockSize - negateBlockSize) * 0.99f));
- }
-
- private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
- long roughBlockSize = maxSize / numBlocks;
- int numEntries = (int) Math.ceil((1.2) * maxSize / roughBlockSize);
- long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP
- + (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY)
- + (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
- long negateBlockSize = totalOverhead / numEntries;
- negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
- return ClassSize.align((long) Math
- .floor((roughBlockSize - negateBlockSize) * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
- }
-
- private static class CachedItem implements Cacheable {
- BlockCacheKey cacheKey;
- int size;
-
- CachedItem(String blockName, int size, int offset) {
- this.cacheKey = new BlockCacheKey(blockName, offset);
- this.size = size;
- }
-
- CachedItem(String blockName, int size) {
- this.cacheKey = new BlockCacheKey(blockName, 0);
- this.size = size;
- }
-
- /** The size of this item reported to the block cache layer */
- @Override
- public long heapSize() {
- return ClassSize.align(size);
- }
-
- /** Size of the cache block holding this item. Used for verification. */
- public long cacheBlockHeapSize() {
- return LruCachedBlock.PER_BLOCK_OVERHEAD + ClassSize.align(cacheKey.heapSize())
- + ClassSize.align(size);
- }
-
- @Override
- public int getSerializedLength() {
- return 0;
- }
-
- @Override
- public CacheableDeserializer getDeserializer() {
- return null;
- }
-
- @Override
- public void serialize(ByteBuffer destination, boolean includeNextBlockMetadata) {
- }
-
- @Override
- public BlockType getBlockType() {
- return BlockType.DATA;
- }
- }
-
- static void testMultiThreadGetAndEvictBlockInternal(BlockCache cache) throws Exception {
- int size = 100;
- int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
- byte[] byteArr = new byte[length];
- HFileContext meta = new HFileContextBuilder().build();
- BlockCacheKey key = new BlockCacheKey("key1", 0);
- HFileBlock blk = new HFileBlock(BlockType.DATA, size, size, -1,
- ByteBuff.wrap(ByteBuffer.wrap(byteArr, 0, size)), HFileBlock.FILL_HEADER, -1, 52, -1, meta,
- HEAP);
- AtomicBoolean err1 = new AtomicBoolean(false);
- Thread t1 = new Thread(() -> {
- for (int i = 0; i < 10000 && !err1.get(); i++) {
- try {
- cache.getBlock(key, false, false, true);
- } catch (Exception e) {
- err1.set(true);
- LOG.info("Cache block or get block failure: ", e);
- }
- }
- });
-
- AtomicBoolean err2 = new AtomicBoolean(false);
- Thread t2 = new Thread(() -> {
- for (int i = 0; i < 10000 && !err2.get(); i++) {
- try {
- cache.evictBlock(key);
- } catch (Exception e) {
- err2.set(true);
- LOG.info("Evict block failure: ", e);
- }
- }
- });
-
- AtomicBoolean err3 = new AtomicBoolean(false);
- Thread t3 = new Thread(() -> {
- for (int i = 0; i < 10000 && !err3.get(); i++) {
- try {
- cache.cacheBlock(key, blk);
- } catch (Exception e) {
- err3.set(true);
- LOG.info("Cache block failure: ", e);
- }
- }
- });
- t1.start();
- t2.start();
- t3.start();
- t1.join();
- t2.join();
- t3.join();
- assertFalse(err1.get());
- assertFalse(err2.get());
- assertFalse(err3.get());
- }
-
- @Test
- public void testMultiThreadGetAndEvictBlock() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
- LruBlockCache cache =
- new LruBlockCache(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
- LruBlockCache.DEFAULT_LOAD_FACTOR, LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
- 0.99f, // acceptable
- 0.33f, // single
- 0.33f, // multi
- 0.34f, // memory
- 1.2f, // limit
- false, 1024);
- testMultiThreadGetAndEvictBlockInternal(cache);
- }
-}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
index 387e2eee31af..0b0b3019119f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -38,6 +39,8 @@
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -101,8 +104,10 @@ public void testScannerSelection() throws IOException {
.build();
RegionInfo info = RegionInfoBuilder.newBuilder(TABLE).build();
+ CacheAccessService cache = CacheAccessServices.fromConfiguration(conf);
HRegion region =
- HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), conf, tableDescriptor);
+ HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), conf,
+ tableDescriptor, cache);
for (int iFile = 0; iFile < NUM_FILES; ++iFile) {
for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
@@ -117,17 +122,17 @@ public void testScannerSelection() throws IOException {
}
Scan scan = new Scan().withStartRow(Bytes.toBytes("aaa")).withStopRow(Bytes.toBytes("aaz"));
- BlockCache cache = BlockCacheFactory.createBlockCache(conf);
InternalScanner scanner = region.getScanner(scan);
List| results = new ArrayList<>();
while (scanner.next(results)) {
}
scanner.close();
assertEquals(0, results.size());
- if (cache instanceof LruBlockCache) {
- Set accessedFiles = ((LruBlockCache) cache).getCachedFileNamesForTest();
- assertEquals(expectedCount, accessedFiles.size());
+ Set accessedFiles = new HashSet<>();
+ for (CachedBlock block : CacheAccessServices.asCachedBlockIterable(cache).orElseThrow()) {
+ accessedFiles.add(block.getFilename());
}
+ assertEquals(expectedCount, accessedFiles.size());
HBaseTestingUtil.closeRegionAndWAL(region);
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
index 8bacfa241571..4cd2fceeabfd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
@@ -24,7 +24,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.LruAdaptiveBlockCache;
-import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
import org.apache.hadoop.hbase.io.hfile.TinyLfuBlockCache;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -36,8 +35,8 @@
*
* These helpers are intended to keep integration-style tests concise while the block cache
* architecture migrates toward {@link CacheAccessService}. They deliberately do not replace
- * implementation-specific tests for {@link LruBlockCache}, {@link LruAdaptiveBlockCache},
- * {@link TinyLfuBlockCache}, {@link BucketCache}, or other concrete cache implementations.
+ * implementation-specific tests for {@link LruAdaptiveBlockCache}, {@link TinyLfuBlockCache},
+ * {@link BucketCache}, or other concrete cache implementations.
*
*
* Each helper mirrors a public constructor on one of the legacy cache implementations and returns
@@ -97,59 +96,59 @@ public static CacheAccessService disabled() {
}
/**
- * Creates a {@link CacheAccessService} backed by {@link LruBlockCache}.
+ * Creates a {@link CacheAccessService} backed by {@link LruCacheEngine}.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
- * @return cache access service backed by {@link LruBlockCache}
+ * @return cache access service backed by {@link LruCacheEngine}
*/
public static CacheAccessService lru(long maxSize, long blockSize) {
return lruInstance(maxSize, blockSize).service();
}
/**
- * Creates a test instance containing {@link LruBlockCache} and its {@link CacheAccessService}
+ * Creates a test instance containing {@link LruCacheEngine} and its {@link CacheAccessService}
* facade.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
* @return test instance
*/
- public static CacheAccessServiceTestInstance lruInstance(long maxSize,
+ public static CacheAccessServiceTestInstance lruInstance(long maxSize,
long blockSize) {
- return new CacheAccessServiceTestInstance<>(new LruBlockCache(maxSize, blockSize));
+ return new CacheAccessServiceTestInstance<>(new LruCacheEngine(maxSize, blockSize));
}
/**
- * Creates a {@link CacheAccessService} backed by {@link LruBlockCache}.
+ * Creates a {@link CacheAccessService} backed by {@link LruCacheEngine}.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
* @param evictionThread whether to start the eviction thread
- * @return cache access service backed by {@link LruBlockCache}
+ * @return cache access service backed by {@link LruCacheEngine}
*/
public static CacheAccessService lru(long maxSize, long blockSize, boolean evictionThread) {
return lruInstance(maxSize, blockSize, evictionThread).service();
}
/**
- * Creates a test instance containing {@link LruBlockCache} and its {@link CacheAccessService}
+ * Creates a test instance containing {@link LruCacheEngine} and its {@link CacheAccessService}
* facade.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
* @param evictionThread whether to start the eviction thread
* @return test instance
*/
- public static CacheAccessServiceTestInstance lruInstance(long maxSize,
+ public static CacheAccessServiceTestInstance lruInstance(long maxSize,
long blockSize, boolean evictionThread) {
return new CacheAccessServiceTestInstance<>(
- new LruBlockCache(maxSize, blockSize, evictionThread));
+ new LruCacheEngine(maxSize, blockSize, evictionThread));
}
/**
- * Creates a {@link CacheAccessService} backed by {@link LruBlockCache}.
+ * Creates a {@link CacheAccessService} backed by {@link LruCacheEngine}.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
* @param evictionThread whether to start the eviction thread
* @param conf configuration used by the cache constructor
- * @return cache access service backed by {@link LruBlockCache}
+ * @return cache access service backed by {@link LruCacheEngine}
* @throws NullPointerException if {@code conf} is {@code null}
*/
public static CacheAccessService lru(long maxSize, long blockSize, boolean evictionThread,
@@ -158,7 +157,7 @@ public static CacheAccessService lru(long maxSize, long blockSize, boolean evict
}
/**
- * Creates a test instance containing {@link LruBlockCache} and its {@link CacheAccessService}
+ * Creates a test instance containing {@link LruCacheEngine} and its {@link CacheAccessService}
* facade.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
@@ -167,19 +166,19 @@ public static CacheAccessService lru(long maxSize, long blockSize, boolean evict
* @return test instance
* @throws NullPointerException if {@code conf} is {@code null}
*/
- public static CacheAccessServiceTestInstance lruInstance(long maxSize,
+ public static CacheAccessServiceTestInstance lruInstance(long maxSize,
long blockSize, boolean evictionThread, Configuration conf) {
Objects.requireNonNull(conf, "conf must not be null");
return new CacheAccessServiceTestInstance<>(
- new LruBlockCache(maxSize, blockSize, evictionThread, conf));
+ new LruCacheEngine(maxSize, blockSize, evictionThread, conf));
}
/**
- * Creates a {@link CacheAccessService} backed by {@link LruBlockCache}.
+ * Creates a {@link CacheAccessService} backed by {@link LruCacheEngine}.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
* @param conf configuration used by the cache constructor
- * @return cache access service backed by {@link LruBlockCache}
+ * @return cache access service backed by {@link LruCacheEngine}
* @throws NullPointerException if {@code conf} is {@code null}
*/
public static CacheAccessService lru(long maxSize, long blockSize, Configuration conf) {
@@ -187,7 +186,7 @@ public static CacheAccessService lru(long maxSize, long blockSize, Configuration
}
/**
- * Creates a test instance containing {@link LruBlockCache} and its {@link CacheAccessService}
+ * Creates a test instance containing {@link LruCacheEngine} and its {@link CacheAccessService}
* facade.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
@@ -195,14 +194,14 @@ public static CacheAccessService lru(long maxSize, long blockSize, Configuration
* @return test instance
* @throws NullPointerException if {@code conf} is {@code null}
*/
- public static CacheAccessServiceTestInstance lruInstance(long maxSize,
+ public static CacheAccessServiceTestInstance lruInstance(long maxSize,
long blockSize, Configuration conf) {
Objects.requireNonNull(conf, "conf must not be null");
- return new CacheAccessServiceTestInstance<>(new LruBlockCache(maxSize, blockSize, conf));
+ return new CacheAccessServiceTestInstance<>(new LruCacheEngine(maxSize, blockSize, conf));
}
/**
- * Creates a {@link CacheAccessService} backed by a fully configured {@link LruBlockCache}.
+ * Creates a {@link CacheAccessService} backed by a fully configured {@link LruCacheEngine}.
* @param maxSize maximum size of this cache, in bytes
* @param blockSize expected average size of blocks, in bytes
* @param evictionThread whether to run evictions in a background thread
@@ -217,7 +216,7 @@ public static CacheAccessServiceTestInstance lruInstance(long max
* @param hardLimitFactor hard capacity limit factor
* @param forceInMemory whether in-memory HFile data blocks have higher priority
* @param maxBlockSize maximum block size accepted by the cache
- * @return cache access service backed by {@link LruBlockCache}
+ * @return cache access service backed by {@link LruCacheEngine}
*/
public static CacheAccessService lru(long maxSize, long blockSize, boolean evictionThread,
int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel, float minFactor,
@@ -229,7 +228,7 @@ public static CacheAccessService lru(long maxSize, long blockSize, boolean evict
}
/**
- * Creates a test instance containing a fully configured {@link LruBlockCache} and its
+ * Creates a test instance containing a fully configured {@link LruCacheEngine} and its
* {@link CacheAccessService} facade.
* @param maxSize maximum size of this cache, in bytes
* @param blockSize expected average size of blocks, in bytes
@@ -247,13 +246,13 @@ public static CacheAccessService lru(long maxSize, long blockSize, boolean evict
* @param maxBlockSize maximum block size accepted by the cache
* @return test instance
*/
- public static CacheAccessServiceTestInstance lruInstance(long maxSize,
+ public static CacheAccessServiceTestInstance lruInstance(long maxSize,
long blockSize, boolean evictionThread, int mapInitialSize, float mapLoadFactor,
int mapConcurrencyLevel, float minFactor, float acceptableFactor, float singleFactor,
float multiFactor, float memoryFactor, float hardLimitFactor, boolean forceInMemory,
long maxBlockSize) {
return new CacheAccessServiceTestInstance<>(
- new LruBlockCache(maxSize, blockSize, evictionThread, mapInitialSize, mapLoadFactor,
+ new LruCacheEngine(maxSize, blockSize, evictionThread, mapInitialSize, mapLoadFactor,
mapConcurrencyLevel, minFactor, acceptableFactor, singleFactor, multiFactor, memoryFactor,
hardLimitFactor, forceInMemory, maxBlockSize));
}
@@ -775,4 +774,10 @@ public static CacheEngine getCacheEngine(CacheAccessService service, CacheTier t
return topologyService.getTopology().getEngine(tier)
.orElseThrow(() -> new IllegalArgumentException("Cache tier is not present: " + tier));
}
+
+ public static CacheAccessService singleTierService(CacheEngine engine) {
+ CacheTopology topology = new SingleTierTopology("single", engine);
+ return new TopologyBackedCacheAccessService(topology,
+ new DefaultHBaseCachePlacementAdmissionPolicy());
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestInstance.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestInstance.java
index b9d8789f3c1d..92a346126191 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestInstance.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestInstance.java
@@ -22,40 +22,83 @@
import org.apache.yetus.audience.InterfaceAudience;
/**
- * Test helper that keeps a concrete {@link BlockCache} together with the {@link CacheAccessService}
- * facade backed by that cache.
+ * Test helper that keeps a concrete cache implementation together with its {@link CacheEngine} and
+ * {@link CacheAccessService} views.
*
- * This is useful for tests that need to exercise production code through
- * {@link CacheAccessService}, while still inspecting implementation-specific state through the
- * concrete cache instance.
+ * The concrete cache may be either a legacy {@link BlockCache} or a native {@link CacheEngine}.
+ * Tests should normally exercise cache operations through {@link CacheAccessService}, while using
+ * the concrete cache or cache engine only for implementation-specific assertions.
*
- * @param concrete block cache type
+ * @param concrete cache implementation type
*/
@InterfaceAudience.Private
-public final class CacheAccessServiceTestInstance {
+public final class CacheAccessServiceTestInstance {
- private final T blockCache;
+ private final T cache;
+ private final BlockCache blockCache;
+ private final CacheEngine cacheEngine;
private final CacheAccessService cacheAccessService;
- CacheAccessServiceTestInstance(T blockCache) {
- this.blockCache = Objects.requireNonNull(blockCache, "blockCache must not be null");
- this.cacheAccessService = CacheAccessServices.fromBlockCache(blockCache);
+ CacheAccessServiceTestInstance(T cache) {
+ this.cache = Objects.requireNonNull(cache, "cache must not be null");
+
+ if (cache instanceof CacheEngine) {
+ this.blockCache = null;
+ this.cacheEngine = (CacheEngine) cache;
+ this.cacheAccessService = createCacheAccessService(cacheEngine);
+ } else if (cache instanceof BlockCache) {
+ this.blockCache = (BlockCache) cache;
+ this.cacheEngine = CacheEngines.fromBlockCache(blockCache);
+ this.cacheAccessService = CacheAccessServices.fromBlockCache(blockCache);
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported cache implementation: " + cache.getClass().getName());
+ }
}
/**
- * Returns the concrete block cache instance.
+ * Creates a single-tier cache access service backed by the specified cache engine.
+ * @param cacheEngine cache engine
+ * @return cache access service
+ */
+ private static CacheAccessService createCacheAccessService(CacheEngine cacheEngine) {
+ Objects.requireNonNull(cacheEngine, "cacheEngine must not be null");
+
+ CacheTopology topology = new SingleTierTopology(cacheEngine.getName(), cacheEngine);
+ return new TopologyBackedCacheAccessService(topology,
+ new DefaultHBaseCachePlacementAdmissionPolicy());
+ }
+
+ /**
+ * Returns the concrete cache implementation supplied to this instance.
+ * @return concrete cache implementation
+ */
+ public T cache() {
+ return cache;
+ }
+
+ /**
+ * Returns the legacy block cache when this instance was constructed from one.
+ * @return legacy block cache, or {@code null} for a native cache engine
+ */
+ public BlockCache blockCache() {
+ return blockCache;
+ }
+
+ /**
+ * Returns the cache engine.
*
- * Tests should use this only when they need implementation-specific inspection or assertions.
- * Normal cache access should go through {@link #service()}.
+ * For a native cache engine this returns the original engine. For a legacy {@link BlockCache},
+ * this returns its cache-engine adapter.
*
- * @return concrete block cache
+ * @return cache engine
*/
- public T blockCache() {
- return blockCache;
+ public CacheEngine cacheEngine() {
+ return cacheEngine;
}
/**
- * Returns the {@link CacheAccessService} facade backed by the concrete block cache.
+ * Returns the cache access service.
* @return cache access service
*/
public CacheAccessService service() {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServiceTestFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServiceTestFactory.java
index ac0b09bee21b..6b064f50e11e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServiceTestFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServiceTestFactory.java
@@ -28,7 +28,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.io.hfile.LruAdaptiveBlockCache;
-import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
import org.apache.hadoop.hbase.io.hfile.TinyLfuBlockCache;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.testclassification.IOTests;
@@ -85,14 +84,14 @@ void testLruServiceOnlyFactory() {
@Test
void testLruInstanceFactory() {
Configuration conf = HBaseConfiguration.create();
- CacheAccessServiceTestInstance instance =
+ CacheAccessServiceTestInstance instance =
CacheAccessServiceTestFactory.lruInstance(MAX_SIZE, BLOCK_SIZE, true, conf);
try {
assertNotNull(instance);
- assertNotNull(instance.blockCache());
+ assertNotNull(instance.cacheEngine());
assertNotNull(instance.service());
- assertInstanceOf(LruBlockCache.class, instance.blockCache());
+ assertInstanceOf(LruCacheEngine.class, instance.cacheEngine());
assertTrue(instance.service().isCacheEnabled());
} finally {
instance.service().shutdown();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestLruCacheEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestLruCacheEngine.java
index 89ef91a62e97..86cb3b4922db 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestLruCacheEngine.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestLruCacheEngine.java
@@ -17,15 +17,47 @@
*/
package org.apache.hadoop.hbase.io.hfile.cache;
+import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheTestUtils;
+import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
+import org.apache.hadoop.hbase.io.hfile.Cacheable;
+import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.io.hfile.LruCachedBlock;
+import org.apache.hadoop.hbase.io.hfile.cache.LruCacheEngine.EvictionThread;
+import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.ClassSize;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests for {@link LruCacheEngine}.
@@ -33,9 +65,891 @@
@Tag(IOTests.TAG)
@Tag(SmallTests.TAG)
public class TestLruCacheEngine {
+ private static final Logger LOG = LoggerFactory.getLogger(TestLruCacheEngine.class);
private static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
+ private static final Configuration CONF = HBaseConfiguration.create();
+
+ @Test
+ public void testCacheEvictionThreadSafe() throws Exception {
+ long maxSize = 100000;
+ int numBlocks = 9;
+ int testRuns = 10;
+ final long blockSize = calculateBlockSizeDefault(maxSize, numBlocks);
+ assertTrue(blockSize * numBlocks <= maxSize, "calculateBlockSize appears broken.");
+
+ final LruCacheEngine cache = new LruCacheEngine(maxSize, blockSize);
+ EvictionThread evictionThread = cache.getEvictionThread();
+ assertTrue(evictionThread != null);
+ Waiter.waitFor(CONF, 10000, 100, () -> evictionThread.isEnteringRun());
+ final String hfileName = "hfile";
+ int threads = 10;
+ final int blocksPerThread = 5 * numBlocks;
+ for (int run = 0; run != testRuns; ++run) {
+ final AtomicInteger blockCount = new AtomicInteger(0);
+ ExecutorService service = Executors.newFixedThreadPool(threads);
+ for (int i = 0; i != threads; ++i) {
+ service.execute(new Runnable() {
+ @Override
+ public void run() {
+ for (int blockIndex = 0; blockIndex < blocksPerThread
+ || (!cache.isEvictionInProgress()); ++blockIndex) {
+ CachedItem block =
+ new CachedItem(hfileName, (int) blockSize, blockCount.getAndIncrement());
+ boolean inMemory = Math.random() > 0.5;
+ cache.cacheBlock(block.cacheKey, block, inMemory);
+ }
+ cache.evictBlocksByHfileName(hfileName);
+ }
+ });
+ }
+ service.shutdown();
+ // The test may fail here if the evict thread frees the blocks too fast
+ service.awaitTermination(10, TimeUnit.MINUTES);
+ Waiter.waitFor(CONF, 10000, 100, new ExplainingPredicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return cache.getBlockCount() == 0;
+ }
+
+ @Override
+ public String explainFailure() throws Exception {
+ return "Cache block count failed to return to 0";
+ }
+ });
+ assertEquals(0, cache.getBlockCount());
+ assertEquals(cache.getOverhead(), cache.getCurrentSize());
+ }
+ }
+
+ @Test
+ public void testBackgroundEvictionThread() throws Exception {
+ long maxSize = 100000;
+ int numBlocks = 9;
+ long blockSize = calculateBlockSizeDefault(maxSize, numBlocks);
+ assertTrue(blockSize * numBlocks <= maxSize, "calculateBlockSize appears broken.");
+
+ LruCacheEngine cache = new LruCacheEngine(maxSize, blockSize);
+ EvictionThread evictionThread = cache.getEvictionThread();
+ assertTrue(evictionThread != null);
+
+ CachedItem[] blocks = generateFixedBlocks(numBlocks + 1, blockSize, "block");
+
+ // Make sure eviction thread has entered run method
+ Waiter.waitFor(CONF, 10000, 10, () -> evictionThread.isEnteringRun());
+
+ // Add all the blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+
+ // wait until at least one eviction has run
+ Waiter.waitFor(CONF, 30000, 200, new ExplainingPredicate() {
+
+ @Override
+ public boolean evaluate() throws Exception {
+ return cache.getStats().getEvictionCount() > 0;
+ }
+
+ @Override
+ public String explainFailure() throws Exception {
+ return "Eviction never happened.";
+ }
+ });
+
+ // let cache stabilize
+ // On some systems, the cache will run multiple evictions before it attains
+ // steady-state. For instance, after populating the cache with 10 blocks,
+ // the first eviction evicts a single block and then a second eviction
+ // evicts another. I think this is due to the delta between minSize and
+ // acceptableSize, combined with variance between object overhead on
+ // different environments.
+ int n = 0;
+ for (long prevCnt = 0 /* < number of blocks added */, curCnt = cache.getBlockCount(); prevCnt
+ != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) {
+ Thread.sleep(200);
+ assertTrue(n++ < 100, "Cache never stabilized.");
+ }
+
+ long evictionCount = cache.getStats().getEvictionCount();
+ assertTrue(evictionCount >= 1);
+ LOG.info("Background Evictions run: {}", evictionCount);
+ }
+
+ @Test
+ public void testCacheSimple() throws Exception {
+ long maxSize = 1000000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 101);
+
+ LruCacheEngine cache = new LruCacheEngine(maxSize, blockSize);
+
+ CachedItem[] blocks = generateRandomBlocks(100, blockSize);
+
+ long expectedCacheSize = cache.heapSize();
+
+ // Confirm empty
+ for (CachedItem block : blocks) {
+ assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null);
+ }
+
+ // Add blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ expectedCacheSize += block.cacheBlockHeapSize();
+ }
+
+ // Verify correctly calculated cache heap size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Check if all blocks are properly cached and retrieved
+ for (CachedItem block : blocks) {
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
+ assertTrue(buf != null);
+ assertEquals(buf.heapSize(), block.heapSize());
+ }
+
+ // Re-add same blocks and ensure nothing has changed
+ long expectedBlockCount = cache.getBlockCount();
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+ assertEquals(expectedBlockCount, cache.getBlockCount(),
+ "Cache should ignore cache requests for blocks already in cache");
+
+ // Verify correctly calculated cache heap size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Check if all blocks are properly cached and retrieved
+ for (CachedItem block : blocks) {
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
+ assertTrue(buf != null);
+ assertEquals(buf.heapSize(), block.heapSize());
+ }
+
+ CacheTestUtils.testConvertToJSON(cache);
+
+ // Expect no evictions
+ assertEquals(0, cache.getStats().getEvictionCount());
+ Thread t = new LruCacheEngine.StatisticsThread(cache);
+ t.start();
+ t.join();
+ }
+
+ @Test
+ public void testCacheEvictionSimple() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 10);
+
+ LruCacheEngine cache = new LruCacheEngine(maxSize, blockSize, false);
+
+ CachedItem[] blocks = generateFixedBlocks(10, blockSize, "block");
+
+ long expectedCacheSize = cache.heapSize();
+
+ // Add all the blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ expectedCacheSize += block.cacheBlockHeapSize();
+ }
+
+ // A single eviction run should have occurred
+ assertEquals(1, cache.getStats().getEvictionCount());
+
+ // Our expected size overruns acceptable limit
+ assertTrue(expectedCacheSize > (maxSize * LruCacheEngine.DEFAULT_ACCEPTABLE_FACTOR));
+
+ // But the cache did not grow beyond max
+ assertTrue(cache.heapSize() < maxSize);
+
+ // And is still below the acceptable limit
+ assertTrue(cache.heapSize() < (maxSize * LruCacheEngine.DEFAULT_ACCEPTABLE_FACTOR));
+
+ // All blocks except block 0 should be in the cache
+ assertTrue(cache.getBlock(blocks[0].cacheKey, true, false, true) == null);
+ for (int i = 1; i < blocks.length; i++) {
+ assertEquals(cache.getBlock(blocks[i].cacheKey, true, false, true), blocks[i]);
+ }
+ }
+
+ @Test
+ public void testCacheEvictionTwoPriorities() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 10);
+
+ LruCacheEngine cache = new LruCacheEngine(maxSize, blockSize, false);
+
+ CachedItem[] singleBlocks = generateFixedBlocks(5, 10000, "single");
+ CachedItem[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
+
+ long expectedCacheSize = cache.heapSize();
+
+ // Add and get the multi blocks
+ for (CachedItem block : multiBlocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ expectedCacheSize += block.cacheBlockHeapSize();
+ assertEquals(cache.getBlock(block.cacheKey, true, false, true), block);
+ }
+
+ // Add the single blocks (no get)
+ for (CachedItem block : singleBlocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ expectedCacheSize += block.heapSize();
+ }
+
+ // A single eviction run should have occurred
+ assertEquals(1, cache.getStats().getEvictionCount());
+
+ // We expect two entries evicted
+ assertEquals(2, cache.getStats().getEvictedCount());
+
+ // Our expected size overruns acceptable limit
+ assertTrue(expectedCacheSize > (maxSize * LruCacheEngine.DEFAULT_ACCEPTABLE_FACTOR));
+
+ // But the cache did not grow beyond max
+ assertTrue(cache.heapSize() <= maxSize);
+
+ // And is now below the acceptable limit
+ assertTrue(cache.heapSize() <= (maxSize * LruCacheEngine.DEFAULT_ACCEPTABLE_FACTOR));
+
+ // We expect fairness across the two priorities.
+ // This test makes multi go barely over its limit, in-memory
+ // empty, and the rest in single. Two single evictions and
+ // one multi eviction expected.
+ assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false, true) == null);
+ assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false, true) == null);
+
+ // And all others to be cached
+ for (int i = 1; i < 4; i++) {
+ assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false, true), singleBlocks[i]);
+ assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false, true), multiBlocks[i]);
+ }
+ }
+
+ @Test
+ public void testCacheEvictionThreePriorities() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.98f, // min
+ 0.99f, // acceptable
+ 0.33f, // single
+ 0.33f, // multi
+ 0.34f, // memory
+ 1.2f, // limit
+ false, 16 * 1024 * 1024);
+
+ CachedItem[] singleBlocks = generateFixedBlocks(5, blockSize, "single");
+ CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
+ CachedItem[] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
+
+ long expectedCacheSize = cache.heapSize();
+
+ // Add 3 blocks from each priority
+ for (int i = 0; i < 3; i++) {
+
+ // Just add single blocks
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
+
+ // Add and get multi blocks
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
+ expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
+ cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
+
+ // Add memory blocks as such
+ cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
+ expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize();
+
+ }
+
+ // Do not expect any evictions yet
+ assertEquals(0, cache.getStats().getEvictionCount());
+
+ // Verify cache size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Insert a single block, oldest single should be evicted
+ cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]);
+
+ // Single eviction, one thing evicted
+ assertEquals(1, cache.getStats().getEvictionCount());
+ assertEquals(1, cache.getStats().getEvictedCount());
+
+ // Verify oldest single block is the one evicted
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
+
+ // Change the oldest remaining single block to a multi
+ cache.getBlock(singleBlocks[1].cacheKey, true, false, true);
+
+ // Insert another single block
+ cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
+
+ // Two evictions, two evicted.
+ assertEquals(2, cache.getStats().getEvictionCount());
+ assertEquals(2, cache.getStats().getEvictedCount());
+
+ // Oldest multi block should be evicted now
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
+
+ // Insert another memory block
+ cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
+
+ // Three evictions, three evicted.
+ assertEquals(3, cache.getStats().getEvictionCount());
+ assertEquals(3, cache.getStats().getEvictedCount());
+
+ // Oldest memory block should be evicted now
+ assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true));
+
+ // Add a block that is twice as big (should force two evictions)
+ CachedItem[] bigBlocks = generateFixedBlocks(3, blockSize * 3, "big");
+ cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]);
+
+ // Four evictions, six evicted (inserted block 3X size, expect +3 evicted)
+ assertEquals(4, cache.getStats().getEvictionCount());
+ assertEquals(6, cache.getStats().getEvictedCount());
+
+ // Expect three remaining singles to be evicted
+ assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true));
+
+ // Make the big block a multi block
+ cache.getBlock(bigBlocks[0].cacheKey, true, false, true);
+
+ // Cache another single big block
+ cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
+
+ // Five evictions, nine evicted (3 new)
+ assertEquals(5, cache.getStats().getEvictionCount());
+ assertEquals(9, cache.getStats().getEvictedCount());
+
+ // Expect three remaining multis to be evicted
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true));
+
+ // Cache a big memory block
+ cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
+
+ // Six evictions, twelve evicted (3 new)
+ assertEquals(6, cache.getStats().getEvictionCount());
+ assertEquals(12, cache.getStats().getEvictedCount());
+
+ // Expect three remaining in-memory to be evicted
+ assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false, true));
+ }
+
+ @Test
+ public void testCacheEvictionInMemoryForceMode() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.98f, // min
+ 0.99f, // acceptable
+ 0.2f, // single
+ 0.3f, // multi
+ 0.5f, // memory
+ 1.2f, // limit
+ true, 16 * 1024 * 1024);
+
+ CachedItem[] singleBlocks = generateFixedBlocks(10, blockSize, "single");
+ CachedItem[] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
+ CachedItem[] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
+
+ long expectedCacheSize = cache.heapSize();
+
+ // 0. Add 5 single blocks and 4 multi blocks to make cache full, si:mu:me = 5:4:0
+ for (int i = 0; i < 4; i++) {
+ // Just add single blocks
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
+ // Add and get multi blocks
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
+ expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
+ cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
+ }
+ // 5th single block
+ cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
+ expectedCacheSize += singleBlocks[4].cacheBlockHeapSize();
+ // Do not expect any evictions yet
+ assertEquals(0, cache.getStats().getEvictionCount());
+ // Verify cache size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // 1. Insert a memory block, oldest single should be evicted, si:mu:me = 4:4:1
+ cache.cacheBlock(memoryBlocks[0].cacheKey, memoryBlocks[0], true);
+ // Single eviction, one block evicted
+ assertEquals(1, cache.getStats().getEvictionCount());
+ assertEquals(1, cache.getStats().getEvictedCount());
+ // Verify oldest single block (index = 0) is the one evicted
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
+
+ // 2. Insert another memory block, another single evicted, si:mu:me = 3:4:2
+ cache.cacheBlock(memoryBlocks[1].cacheKey, memoryBlocks[1], true);
+ // Two evictions, two evicted.
+ assertEquals(2, cache.getStats().getEvictionCount());
+ assertEquals(2, cache.getStats().getEvictedCount());
+ // Current oldest single block (index = 1) should be evicted now
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
+
+ // 3. Insert 4 memory blocks, 2 single and 2 multi evicted, si:mu:me = 1:2:6
+ cache.cacheBlock(memoryBlocks[2].cacheKey, memoryBlocks[2], true);
+ cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
+ cache.cacheBlock(memoryBlocks[4].cacheKey, memoryBlocks[4], true);
+ cache.cacheBlock(memoryBlocks[5].cacheKey, memoryBlocks[5], true);
+ // Three evictions, three evicted.
+ assertEquals(6, cache.getStats().getEvictionCount());
+ assertEquals(6, cache.getStats().getEvictedCount());
+ // two oldest single blocks and two oldest multi blocks evicted
+ assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
+
+ // 4. Insert 3 memory blocks, the remaining 1 single and 2 multi evicted
+ // si:mu:me = 0:0:9
+ cache.cacheBlock(memoryBlocks[6].cacheKey, memoryBlocks[6], true);
+ cache.cacheBlock(memoryBlocks[7].cacheKey, memoryBlocks[7], true);
+ cache.cacheBlock(memoryBlocks[8].cacheKey, memoryBlocks[8], true);
+ // Three evictions, three evicted.
+ assertEquals(9, cache.getStats().getEvictionCount());
+ assertEquals(9, cache.getStats().getEvictedCount());
+ // one oldest single block and two oldest multi blocks evicted
+ assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[3].cacheKey, true, false, true));
+
+ // 5. Insert one memory block, the oldest memory evicted
+ // si:mu:me = 0:0:9
+ cache.cacheBlock(memoryBlocks[9].cacheKey, memoryBlocks[9], true);
+ // one eviction, one evicted.
+ assertEquals(10, cache.getStats().getEvictionCount());
+ assertEquals(10, cache.getStats().getEvictedCount());
+ // oldest memory block evicted
+ assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true));
+
+ // 6. Insert one new single block, itself evicted immediately since
+ // all blocks in cache are memory-type which have higher priority
+ // si:mu:me = 0:0:9 (no change)
+ cache.cacheBlock(singleBlocks[9].cacheKey, singleBlocks[9]);
+ // one eviction, one evicted.
+ assertEquals(11, cache.getStats().getEvictionCount());
+ assertEquals(11, cache.getStats().getEvictedCount());
+ // the single block just cached now evicted (can't evict memory)
+ assertEquals(null, cache.getBlock(singleBlocks[9].cacheKey, true, false, true));
+ }
+
+ // test scan resistance
+ @Test
+ public void testScanResistance() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
+ 0.99f, // acceptable
+ 0.33f, // single
+ 0.33f, // multi
+ 0.34f, // memory
+ 1.2f, // limit
+ false, 16 * 1024 * 1024);
+
+ CachedItem[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
+ CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
+
+ // Add 5 multi blocks
+ for (CachedItem block : multiBlocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ cache.getBlock(block.cacheKey, true, false, true);
+ }
+
+ // Add 5 single blocks
+ for (int i = 0; i < 5; i++) {
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ }
+
+ // An eviction ran
+ assertEquals(1, cache.getStats().getEvictionCount());
+
+ // To drop down to 2/3 capacity, we'll need to evict 4 blocks
+ assertEquals(4, cache.getStats().getEvictedCount());
+
+ // Should have been taken off equally from single and multi
+ assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
+
+ // Let's keep "scanning" by adding single blocks. From here on we only
+ // expect evictions from the single bucket.
+
+ // Every time we reach 10 total blocks (every 4 inserts) we get 4 single
+ // blocks evicted. Inserting 13 blocks should yield 3 more evictions and
+ // 12 more evicted.
+
+ for (int i = 5; i < 18; i++) {
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ }
+
+ // 4 total evictions, 16 total evicted
+ assertEquals(4, cache.getStats().getEvictionCount());
+ assertEquals(16, cache.getStats().getEvictedCount());
+
+ // Should now have 7 total blocks
+ assertEquals(7, cache.getBlockCount());
+
+ }
+
+ @Test
+ public void testMaxBlockSize() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
+ 0.99f, // acceptable
+ 0.33f, // single
+ 0.33f, // multi
+ 0.34f, // memory
+ 1.2f, // limit
+ false, 1024);
+ CachedItem[] tooLong = generateFixedBlocks(10, 1024 + 5, "long");
+ CachedItem[] small = generateFixedBlocks(15, 600, "small");
+
+ for (CachedItem i : tooLong) {
+ cache.cacheBlock(i.cacheKey, i);
+ }
+ for (CachedItem i : small) {
+ cache.cacheBlock(i.cacheKey, i);
+ }
+ assertEquals(15, cache.getBlockCount());
+ for (CachedItem i : small) {
+ assertNotNull(cache.getBlock(i.cacheKey, true, false, false));
+ }
+ for (CachedItem i : tooLong) {
+ assertNull(cache.getBlock(i.cacheKey, true, false, false));
+ }
+
+ assertEquals(10, cache.getStats().getFailedInserts());
+ }
+
+ // test setMaxSize
+ @Test
+ public void testResizeBlockCache() throws Exception {
+ long maxSize = 300000;
+ long blockSize = calculateBlockSize(maxSize, 31);
+
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.98f, // min
+ 0.99f, // acceptable
+ 0.33f, // single
+ 0.33f, // multi
+ 0.34f, // memory
+ 1.2f, // limit
+ false, 16 * 1024 * 1024);
+
+ CachedItem[] singleBlocks = generateFixedBlocks(10, blockSize, "single");
+ CachedItem[] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
+ CachedItem[] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
+
+ // Add all blocks from all priorities
+ for (int i = 0; i < 10; i++) {
+ // Just add single blocks
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+
+ // Add and get multi blocks
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
+ cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
+
+ // Add memory blocks as such
+ cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
+ }
+
+ // Do not expect any evictions yet
+ assertEquals(0, cache.getStats().getEvictionCount());
+
+ // Resize to half capacity plus an extra block (otherwise we evict an extra)
+ cache.setMaxSize((long) (maxSize * 0.5f));
+
+ // Should have run a single eviction
+ assertEquals(1, cache.getStats().getEvictionCount());
+
+ // And we expect 1/2 of the blocks to be evicted
+ assertEquals(15, cache.getStats().getEvictedCount());
+
+ // And the oldest 5 blocks from each category should be gone
+ for (int i = 0; i < 5; i++) {
+ assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false, true));
+ assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
+ }
+
+ // And the newest 5 blocks should still be accessible
+ for (int i = 5; i < 10; i++) {
+ assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false, true));
+ assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false, true));
+ assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
+ }
+ }
+
+ // test metricsPastNPeriods
+ @Test
+ public void testPastNPeriodsMetrics() throws Exception {
+ double delta = 0.01;
+
+ // 3 total periods
+ CacheStats stats = new CacheStats("test", 3);
+
+ // No accesses, should be 0
+ stats.rollMetricsPeriod();
+ assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 1, 1 hit caching, 1 hit non-caching, 2 miss non-caching
+ // should be (2/4)=0.5 and (1/1)=1
+ stats.hit(false, true, BlockType.DATA);
+ stats.hit(true, true, BlockType.DATA);
+ stats.miss(false, false, BlockType.DATA);
+ stats.miss(false, false, BlockType.DATA);
+ stats.rollMetricsPeriod();
+ assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 2, 1 miss caching, 3 miss non-caching
+ // should be (2/8)=0.25 and (1/2)=0.5
+ stats.miss(true, false, BlockType.DATA);
+ stats.miss(false, false, BlockType.DATA);
+ stats.miss(false, false, BlockType.DATA);
+ stats.miss(false, false, BlockType.DATA);
+ stats.rollMetricsPeriod();
+ assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 3, 2 hits of each type
+ // should be (6/12)=0.5 and (3/4)=0.75
+ stats.hit(false, true, BlockType.DATA);
+ stats.hit(true, true, BlockType.DATA);
+ stats.hit(false, true, BlockType.DATA);
+ stats.hit(true, true, BlockType.DATA);
+ stats.rollMetricsPeriod();
+ assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 4, evict period 1, two caching misses
+ // should be (4/10)=0.4 and (2/5)=0.4
+ stats.miss(true, false, BlockType.DATA);
+ stats.miss(true, false, BlockType.DATA);
+ stats.rollMetricsPeriod();
+ assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 5, evict period 2, 2 caching misses, 2 non-caching hit
+ // should be (6/10)=0.6 and (2/6)=1/3
+ stats.miss(true, false, BlockType.DATA);
+ stats.miss(true, false, BlockType.DATA);
+ stats.hit(false, true, BlockType.DATA);
+ stats.hit(false, true, BlockType.DATA);
+ stats.rollMetricsPeriod();
+ assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals((double) 1 / 3, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 6, evict period 3
+ // should be (2/6)=1/3 and (0/4)=0
+ stats.rollMetricsPeriod();
+ assertEquals((double) 1 / 3, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 7, evict period 4
+ // should be (2/4)=0.5 and (0/2)=0
+ stats.rollMetricsPeriod();
+ assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 8, evict period 5
+ // should be 0 and 0
+ stats.rollMetricsPeriod();
+ assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+ // period 9, one of each
+ // should be (2/4)=0.5 and (1/2)=0.5
+ stats.miss(true, false, BlockType.DATA);
+ stats.miss(false, false, BlockType.DATA);
+ stats.hit(true, true, BlockType.DATA);
+ stats.hit(false, true, BlockType.DATA);
+ stats.rollMetricsPeriod();
+ assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+ assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
+ }
+
+ @Test
+ public void testCacheBlockNextBlockMetadataMissing() {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+ int size = 100;
+ int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
+ byte[] byteArr = new byte[length];
+ ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
+ HFileContext meta = new HFileContextBuilder().build();
+ HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1,
+ ByteBuff.wrap(buf), HFileBlock.FILL_HEADER, -1, 52, -1, meta, HEAP);
+ HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1,
+ ByteBuff.wrap(buf), HFileBlock.FILL_HEADER, -1, -1, -1, meta, HEAP);
+
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
+ 0.99f, // acceptable
+ 0.33f, // single
+ 0.33f, // multi
+ 0.34f, // memory
+ 1.2f, // limit
+ false, 1024);
+
+ BlockCacheKey key = new BlockCacheKey("key1", 0);
+ ByteBuffer actualBuffer = ByteBuffer.allocate(length);
+ ByteBuffer block1Buffer = ByteBuffer.allocate(length);
+ ByteBuffer block2Buffer = ByteBuffer.allocate(length);
+ blockWithNextBlockMetadata.serialize(block1Buffer, true);
+ blockWithoutNextBlockMetadata.serialize(block2Buffer, true);
+
+ // Add blockWithNextBlockMetadata, expect blockWithNextBlockMetadata back.
+ CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata, actualBuffer,
+ block1Buffer);
+
+ // Add blockWithoutNextBlockMetada, expect blockWithNextBlockMetadata back.
+ CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer,
+ block1Buffer);
+
+ // Clear and add blockWithoutNextBlockMetadata
+ cache.clearCache();
+ assertNull(cache.getBlock(key, false, false, false));
+ CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer,
+ block2Buffer);
+
+ // Add blockWithNextBlockMetadata, expect blockWithNextBlockMetadata to replace.
+ CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata, actualBuffer,
+ block1Buffer);
+ }
+
+ private CachedItem[] generateFixedBlocks(int numBlocks, int size, String pfx) {
+ CachedItem[] blocks = new CachedItem[numBlocks];
+ for (int i = 0; i < numBlocks; i++) {
+ blocks[i] = new CachedItem(pfx + i, size);
+ }
+ return blocks;
+ }
+
+ private CachedItem[] generateFixedBlocks(int numBlocks, long size, String pfx) {
+ return generateFixedBlocks(numBlocks, (int) size, pfx);
+ }
+
+ private CachedItem[] generateRandomBlocks(int numBlocks, long maxSize) {
+ CachedItem[] blocks = new CachedItem[numBlocks];
+ Random rand = ThreadLocalRandom.current();
+ for (int i = 0; i < numBlocks; i++) {
+ blocks[i] = new CachedItem("block" + i, rand.nextInt((int) maxSize) + 1);
+ }
+ return blocks;
+ }
+
+ private long calculateBlockSize(long maxSize, int numBlocks) {
+ long roughBlockSize = maxSize / numBlocks;
+ int numEntries = (int) Math.ceil((1.2) * maxSize / roughBlockSize);
+ long totalOverhead = LruCacheEngine.CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP
+ + (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY)
+ + (LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+ long negateBlockSize = (long) (totalOverhead / numEntries);
+ negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
+ return ClassSize.align((long) Math.floor((roughBlockSize - negateBlockSize) * 0.99f));
+ }
+
+ private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
+ long roughBlockSize = maxSize / numBlocks;
+ int numEntries = (int) Math.ceil((1.2) * maxSize / roughBlockSize);
+ long totalOverhead = LruCacheEngine.CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP
+ + (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY)
+ + (LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+ long negateBlockSize = totalOverhead / numEntries;
+ negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
+ return ClassSize.align((long) Math
+ .floor((roughBlockSize - negateBlockSize) * LruCacheEngine.DEFAULT_ACCEPTABLE_FACTOR));
+ }
+
+ private static class CachedItem implements Cacheable {
+ BlockCacheKey cacheKey;
+ int size;
+
+ CachedItem(String blockName, int size, int offset) {
+ this.cacheKey = new BlockCacheKey(blockName, offset);
+ this.size = size;
+ }
+
+ CachedItem(String blockName, int size) {
+ this.cacheKey = new BlockCacheKey(blockName, 0);
+ this.size = size;
+ }
+
+ /** The size of this item reported to the block cache layer */
+ @Override
+ public long heapSize() {
+ return ClassSize.align(size);
+ }
+
+ /** Size of the cache block holding this item. Used for verification. */
+ public long cacheBlockHeapSize() {
+ return LruCachedBlock.PER_BLOCK_OVERHEAD + ClassSize.align(cacheKey.heapSize())
+ + ClassSize.align(size);
+ }
+
+ @Override
+ public int getSerializedLength() {
+ return 0;
+ }
+
+ @Override
+ public CacheableDeserializer getDeserializer() {
+ return null;
+ }
+
+ @Override
+ public void serialize(ByteBuffer destination, boolean includeNextBlockMetadata) {
+ }
+
+ @Override
+ public BlockType getBlockType() {
+ return BlockType.DATA;
+ }
+ }
+
+ @Test
+ public void testMultiThreadGetAndEvictBlock() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+ LruCacheEngine cache =
+ new LruCacheEngine(maxSize, blockSize, false, (int) Math.ceil(1.2 * maxSize / blockSize),
+ LruCacheEngine.DEFAULT_LOAD_FACTOR, LruCacheEngine.DEFAULT_CONCURRENCY_LEVEL, 0.66f, // min
+ 0.99f, // acceptable
+ 0.33f, // single
+ 0.33f, // multi
+ 0.34f, // memory
+ 1.2f, // limit
+ false, 1024);
+ BlockCacheTestUtils.testMultiThreadGetAndEvictBlockInternal(cache);
+ }
+
/**
* Verifies that an eviction pass with no excess occupancy reports zero cache pressure.
*/
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
index 0d200ec54afd..83ddb090220f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
@@ -29,8 +29,12 @@
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
+import org.apache.hadoop.hbase.io.hfile.CachedBlock;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices;
+import org.apache.hadoop.util.Lists;
/**
* Test seek performance for encoded data blocks. Read an HFile and do several random seeks.
@@ -179,6 +183,15 @@ private static void printUsage() {
}
private void clearBlockCache() {
- ((LruBlockCache) cacheConf.getBlockCache().get()).clearCache();
+ CacheAccessService cache = cacheConf.getCacheAccessService();
+ Iterable cachedBlocks =
+ CacheAccessServices.asCachedBlockIterable(cache).orElseThrow();
+
+ for (CachedBlock block : Lists.newArrayList(cachedBlocks)) {
+ BlockCacheKey key = new BlockCacheKey(block.getFilename(), block.getOffset());
+ while (cache.evictBlock(key)) {
+ // A block may be present in more than one cache tier.
+ }
+ }
}
}
|