HBASE-26975 Add on heap and off heap memstore info in rs web UI (#4368)

Signed-off-by: Duo Zhang <[email protected]>

Author: liangxs

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -101,6 +101,16 @@ public interface MetricsRegionServerWrapper {
    */
   long getMemStoreSize();
 
+  /**
+   * Get the size of the on heap memstore on this region server.
+   */
+  long getOnHeapMemStoreSize();
+
+  /**
+   * Get the size of the off heap memstore on this region server.
+   */
+  long getOffHeapMemStoreSize();
+
   /**
    * Get the total size of the store files this region server is serving from.
    */
@@ -236,7 +246,22 @@ public interface MetricsRegionServerWrapper {
    */
   int getFlushQueueSize();
 
+  /**
+   * Get the limit size of the off heap memstore (if enabled), otherwise
+   * get the limit size of the on heap memstore.
+   */
   long getMemStoreLimit();
+
+  /**
+   * Get the limit size of the on heap memstore.
+   */
+  long getOnHeapMemStoreLimit();
+
+  /**
+   * Get the limit size of the off heap memstore.
+   */
+  long getOffHeapMemStoreLimit();
+
   /**
    * Get the size (in bytes) of the block cache that is free.
    */

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon

@@ -113,8 +113,9 @@ MetricsRegionServerWrapper mWrap;
         <th>Max Heap</th>
         <th>Direct Memory Used</th>
         <th>Direct Memory Configured</th>
-        <th>Memstore Size</th>
-        <th>Memstore Limit</th>
+        <th>Memstore On-Heap Size / Limit</th>
+        <th>Memstore Off-Heap Size / Limit</th>
+        <th>Memstore Data Size (On&&Off Heap)</th>
     </tr>
 </tr>
 <tr>
@@ -131,10 +132,15 @@ MetricsRegionServerWrapper mWrap;
         <% TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemorySize(), "B", 1) %>
     </td>
     <td>
-        <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %>
+        <% TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreSize(), "B", 1) + " / "
+         + TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreLimit(), "B", 1) %>
+    </td>
+    <td>
+        <% TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreSize(), "B", 1) + " / "
+         + TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreLimit(), "B", 1) %>
     </td>
     <td>
-        <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreLimit(), "B", 1) %>
+        <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %>
     </td>
 </tr>
 </table>

hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java

@@ -30,7 +30,7 @@
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
- * Util class to calculate memory size for memstore, block cache(L1, L2) of RS.
+ * Util class to calculate memory size for memstore(on heap, off heap), block cache(L1, L2) of RS.
  */
 @InterfaceAudience.Private
 public class MemorySizeUtil {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -77,6 +77,8 @@ class MetricsRegionServerWrapperImpl
   private volatile long walFileSize = 0;
   private volatile long numStoreFiles = 0;
   private volatile long memstoreSize = 0;
+  private volatile long onHeapMemstoreSize = 0;
+  private volatile long offHeapMemstoreSize = 0;
   private volatile long storeFileSize = 0;
   private volatile double storeFileSizeGrowthRate = 0;
   private volatile long maxStoreFileAge = 0;
@@ -282,6 +284,16 @@ public long getMemStoreLimit() {
     return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
   }
 
+  @Override
+  public long getOnHeapMemStoreLimit() {
+    return this.regionServer.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit();
+  }
+
+  @Override
+  public long getOffHeapMemStoreLimit() {
+    return this.regionServer.getRegionServerAccounting().getGlobalOffHeapMemStoreLimit();
+  }
+
   @Override
   public long getBlockCacheSize() {
     return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
@@ -450,6 +462,16 @@ public long getMemStoreSize() {
     return memstoreSize;
   }
 
+  @Override
+  public long getOnHeapMemStoreSize() {
+    return onHeapMemstoreSize;
+  }
+
+  @Override
+  public long getOffHeapMemStoreSize() {
+    return offHeapMemstoreSize;
+  }
+
   @Override
   public long getStoreFileSize() {
     return storeFileSize;
@@ -695,7 +717,8 @@ synchronized public void run() {
         HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
             new HDFSBlocksDistribution();
 
-        long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
+        long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0;
+        long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0;
         long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
         long avgAgeNumerator = 0, numHFiles = 0;
         long tempMinStoreFileAge = Long.MAX_VALUE;
@@ -776,6 +799,8 @@ synchronized public void run() {
           for (Store store : storeList) {
             tempNumStoreFiles += store.getStorefilesCount();
             tempMemstoreSize += store.getMemStoreSize().getDataSize();
+            tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize();
+            tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize();
             tempStoreFileSize += store.getStorefilesSize();
 
             OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -877,6 +902,8 @@ synchronized public void run() {
         numStores = tempNumStores;
         numStoreFiles = tempNumStoreFiles;
         memstoreSize = tempMemstoreSize;
+        onHeapMemstoreSize = tempOnHeapMemstoreSize;
+        offHeapMemstoreSize = tempOffHeapMemstoreSize;
         storeFileSize = tempStoreFileSize;
         maxStoreFileAge = tempMaxStoreFileAge;
         if (regionCount > 0) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java

@@ -81,6 +81,14 @@ long getGlobalMemStoreLimit() {
     return this.globalMemStoreLimit;
   }
 
+  long getGlobalOffHeapMemStoreLimit() {
+    if (isOffheap()) {
+      return this.globalMemStoreLimit;
+    } else {
+      return 0;
+    }
+  }
+
   long getGlobalOnHeapMemStoreLimit() {
     return this.globalOnHeapMemstoreLimit;
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -68,6 +68,16 @@ public long getMemStoreSize() {
     return 1025;
   }
 
+  @Override
+  public long getOnHeapMemStoreSize() {
+    return 500;
+  }
+
+  @Override
+  public long getOffHeapMemStoreSize() {
+    return 600;
+  }
+
   @Override
   public long getStoreFileSize() {
     return 1900;
@@ -263,6 +273,16 @@ public long getMemStoreLimit() {
     return 419;
   }
 
+  @Override
+  public long getOnHeapMemStoreLimit() {
+    return 311;
+  }
+
+  @Override
+  public long getOffHeapMemStoreLimit() {
+    return 419;
+  }
+
   @Override
   public long getBlockCacheFreeSize() {
     return 413;