HBASE-26835 Rewrite TestLruAdaptiveBlockCache to make it more stable (#4219)

Signed-off-by: Xiaolin Ha <[email protected]>

Author: Apache9

hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java

@@ -66,6 +66,8 @@ public class TestLruAdaptiveBlockCache {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestLruAdaptiveBlockCache.class);
 
+  private static final Configuration CONF = HBaseConfiguration.create();
+
   @Test
   public void testCacheEvictionThreadSafe() throws Exception {
     long maxSize = 100000;
@@ -74,13 +76,10 @@ public void testCacheEvictionThreadSafe() throws Exception {
     final long blockSize = calculateBlockSizeDefault(maxSize, numBlocks);
     assertTrue("calculateBlockSize appears broken.", blockSize * numBlocks <= maxSize);
 
-    final Configuration conf = HBaseConfiguration.create();
     final LruAdaptiveBlockCache cache = new LruAdaptiveBlockCache(maxSize, blockSize);
     EvictionThread evictionThread = cache.getEvictionThread();
     assertNotNull(evictionThread);
-    while (!evictionThread.isEnteringRun()) {
-      Thread.sleep(1000);
-    }
+    Waiter.waitFor(CONF, 10000, 100, () -> evictionThread.isEnteringRun());
     final String hfileName = "hfile";
     int threads = 10;
     final int blocksPerThread = 5 * numBlocks;
@@ -102,7 +101,7 @@ public void testCacheEvictionThreadSafe() throws Exception {
       service.shutdown();
       // The test may fail here if the evict thread frees the blocks too fast
       service.awaitTermination(10, TimeUnit.MINUTES);
-      Waiter.waitFor(conf, 10000, 100, new ExplainingPredicate<Exception>() {
+      Waiter.waitFor(CONF, 10000, 100, new ExplainingPredicate<Exception>() {
         @Override
         public boolean evaluate() throws Exception {
           return cache.getBlockCount() == 0;
@@ -133,21 +132,26 @@ public void testBackgroundEvictionThread() throws Exception {
     CachedItem[] blocks = generateFixedBlocks(numBlocks + 1, blockSize, "block");
 
     // Make sure eviction thread has entered run method
-    while (!evictionThread.isEnteringRun()) {
-      Thread.sleep(1);
-    }
+    Waiter.waitFor(CONF, 10000, 10, () -> evictionThread.isEnteringRun());
 
     // Add all the blocks
     for (CachedItem block : blocks) {
       cache.cacheBlock(block.cacheKey, block);
     }
 
     // wait until at least one eviction has run
-    int n = 0;
-    while(cache.getStats().getEvictionCount() == 0) {
-      Thread.sleep(200);
-      assertTrue("Eviction never happened.", n++ < 20);
-    }
+    Waiter.waitFor(CONF, 30000, 200, new ExplainingPredicate<Exception>() {
+
+      @Override
+      public boolean evaluate() throws Exception {
+        return cache.getStats().getEvictionCount() > 0;
+      }
+
+      @Override
+      public String explainFailure() throws Exception {
+        return "Eviction never happened.";
+      }
+    });
 
     // let cache stabilize
     // On some systems, the cache will run multiple evictions before it attains
@@ -156,22 +160,20 @@ public void testBackgroundEvictionThread() throws Exception {
     // evicts another. I think this is due to the delta between minSize and
     // acceptableSize, combined with variance between object overhead on
     // different environments.
-    n = 0;
-    for (long prevCnt = 0 /* < number of blocks added */,
-         curCnt = cache.getBlockCount();
-         prevCnt != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) {
+    int n = 0;
+    for (long prevCnt = 0 /* < number of blocks added */, curCnt =
+      cache.getBlockCount(); prevCnt != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) {
       Thread.sleep(200);
-      assertTrue("Cache never stabilized.", n++ < 20);
+      assertTrue("Cache never stabilized.", n++ < 100);
     }
 
     long evictionCount = cache.getStats().getEvictionCount();
     assertTrue(evictionCount >= 1);
-    System.out.println("Background Evictions run: " + evictionCount);
+    LOG.info("Background Evictions run: {}", evictionCount);
   }
 
   @Test
   public void testCacheSimple() throws Exception {
-
     long maxSize = 1000000;
     long blockSize = calculateBlockSizeDefault(maxSize, 101);
 
@@ -233,7 +235,6 @@ public void testCacheSimple() throws Exception {
 
   @Test
   public void testCacheEvictionSimple() throws Exception {
-
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 10);
 
@@ -275,14 +276,13 @@ public void testCacheEvictionSimple() throws Exception {
 
   @Test
   public void testCacheEvictionTwoPriorities() throws Exception {
-
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 10);
 
     LruAdaptiveBlockCache cache = new LruAdaptiveBlockCache(maxSize,blockSize,false);
 
-    CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single");
-    CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi");
+    CachedItem[] singleBlocks = generateFixedBlocks(5, 10000, "single");
+    CachedItem[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
 
     long expectedCacheSize = cache.heapSize();
 
@@ -339,7 +339,6 @@ public void testCacheEvictionTwoPriorities() throws Exception {
 
   @Test
   public void testCacheEvictionThreePriorities() throws Exception {
-
     long maxSize = 100000;
     long blockSize = calculateBlockSize(maxSize, 10);
 
@@ -359,9 +358,9 @@ public void testCacheEvictionThreePriorities() throws Exception {
       500,
       0.01f);
 
-    CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single");
-    CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
-    CachedItem [] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
+    CachedItem[] singleBlocks = generateFixedBlocks(5, blockSize, "single");
+    CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
+    CachedItem[] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
 
     long expectedCacheSize = cache.heapSize();
 
@@ -617,8 +616,8 @@ public void testScanResistance() throws Exception {
       500,
       0.01f);
 
-    CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
-    CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
+    CachedItem[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
+    CachedItem[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
 
     // Add 5 multi blocks
     for (CachedItem block : multiBlocks) {
@@ -627,7 +626,7 @@ public void testScanResistance() throws Exception {
     }
 
     // Add 5 single blocks
-    for(int i=0;i<5;i++) {
+    for (int i = 0; i < 5; i++) {
       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
     }
 
@@ -654,7 +653,7 @@ public void testScanResistance() throws Exception {
     // blocks evicted.  Inserting 13 blocks should yield 3 more evictions and
     // 12 more evicted.
 
-    for(int i=5;i<18;i++) {
+    for (int i = 5; i < 18; i++) {
       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
     }
 
@@ -688,9 +687,8 @@ public void testMaxBlockSize() throws Exception {
       500,
       0.01f);
 
-    CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long");
-    CachedItem [] small = generateFixedBlocks(15, 600, "small");
-
+    CachedItem[] tooLong = generateFixedBlocks(10, 1024+5, "long");
+    CachedItem[] small = generateFixedBlocks(15, 600, "small");
 
     for (CachedItem i:tooLong) {
       cache.cacheBlock(i.cacheKey, i);
@@ -712,7 +710,6 @@ public void testMaxBlockSize() throws Exception {
   // test setMaxSize
   @Test
   public void testResizeBlockCache() throws Exception {
-
     long maxSize = 300000;
     long blockSize = calculateBlockSize(maxSize, 31);
 
@@ -732,13 +729,12 @@ public void testResizeBlockCache() throws Exception {
       500,
       0.01f);
 
-    CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single");
-    CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
-    CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
+    CachedItem[] singleBlocks = generateFixedBlocks(10, blockSize, "single");
+    CachedItem[] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
+    CachedItem[] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
 
     // Add all blocks from all priorities
-    for(int i=0;i<10;i++) {
-
+    for (int i = 0; i < 10; i++) {
       // Just add single blocks
       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
 
@@ -939,11 +935,11 @@ public void testCacheBlockNextBlockMetadataMissing() {
     return blocks;
   }
 
-  private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
+  private CachedItem[] generateFixedBlocks(int numBlocks, long size, String pfx) {
     return generateFixedBlocks(numBlocks, (int)size, pfx);
   }
 
-  private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
+  private CachedItem[] generateRandomBlocks(int numBlocks, long maxSize) {
     CachedItem [] blocks = new CachedItem[numBlocks];
     Random rand = ThreadLocalRandom.current();
     for(int i=0;i<numBlocks;i++) {
@@ -1126,9 +1122,7 @@ public void testSkipCacheDataBlocksInteral(int heavyEvictionCountLimit) throws E
 
     EvictionThread evictionThread = cache.getEvictionThread();
     assertNotNull(evictionThread);
-    while (!evictionThread.isEnteringRun()) {
-      Thread.sleep(1);
-    }
+    Waiter.waitFor(CONF, 10000, 10, () -> evictionThread.isEnteringRun());
 
     final String hfileName = "hfile";
     for (int blockIndex = 0; blockIndex <= numBlocks * 3000; ++blockIndex) {