YARN-9505. Add container allocation latency for Opportunistic Scheduler. Contributed by Abhishek Modi.

Author: Giovanni Matteo Fumarola

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/OpportunisticSchedulerMetrics.java

@@ -28,6 +28,7 @@
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
 import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -86,6 +87,9 @@ private static void registerMetrics() {
   @Metric("Aggregate # of allocated off-switch opportunistic containers")
   MutableCounterLong aggregateOffSwitchOContainersAllocated;
 
+  @Metric("Aggregate latency for opportunistic container allocation")
+  MutableQuantiles allocateLatencyOQuantiles;
+
   @VisibleForTesting
   public int getAllocatedContainers() {
     return allocatedOContainers.value();
@@ -138,4 +142,8 @@ public void incrRackLocalOppContainers() {
   public void incrOffSwitchOppContainers() {
     aggregateOffSwitchOContainersAllocated.incr();
   }
+
+  public void addAllocateOLatencyEntry(long latency) {
+    allocateLatencyOQuantiles.add(latency);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java

@@ -22,6 +22,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -241,9 +242,15 @@ static class EnrichedResourceRequest {
     private final Map<String, AtomicInteger> nodeLocations = new HashMap<>();
     private final Map<String, AtomicInteger> rackLocations = new HashMap<>();
     private final ResourceRequest request;
+    private final long timestamp;
 
     EnrichedResourceRequest(ResourceRequest request) {
       this.request = request;
+      timestamp = Time.monotonicNow();
+    }
+
+    long getTimestamp() {
+      return timestamp;
     }
 
     ResourceRequest getRequest() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java

@@ -18,9 +18,12 @@
 
 package org.apache.hadoop.yarn.server.scheduler;
 
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RemoteNode;
+import org.apache.hadoop.yarn.server.metrics.OpportunisticSchedulerMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -191,7 +194,14 @@ public void matchAllocationToOutstandingRequest(Resource capability,
             err.removeLocation(allocation.getResourceName());
           }
         }
+        getOppSchedulerMetrics().addAllocateOLatencyEntry(
+            Time.monotonicNow() - err.getTimestamp());
       }
     }
   }
+
+  @VisibleForTesting
+  OpportunisticSchedulerMetrics getOppSchedulerMetrics() {
+    return OpportunisticSchedulerMetrics.getMetrics();
+  }
 }