Merge pull request #10 from awslabs/metrics_histogram_fixes

Refactoring metrics histogram and step histogram

Author: Vikas-kum

examples/profiler/profiler_generic_dashboard.ipynb

@@ -171,6 +171,9 @@ def __init__(self, s3_trial_path, use_in_memory_cache=False):
             self.bucket_name = bucket_name
             self.base_folder = base_folder
             self.prefix = os.path.join(self.base_folder, self.prefix, "")
+            self.logger.info(
+                f"S3AlgorithmMetricsReader created with bucket:{bucket_name} and prefix:{self.prefix}"
+            )
         # Pre-build the file list so that user can query get_timestamp_of_latest_available_file() and get_current_time_range_for_event_query
         self.refresh_event_file_list()
 
@@ -202,9 +205,9 @@ def parse_event_files(self, event_files):
     """
 
     def refresh_event_file_list(self):
-        list_dir = ListRequest(
-            Bucket=self.bucket_name,
-            Prefix=self.prefix,
-            StartAfter=self._startAfter_prefix if self._startAfter_prefix else self.prefix,
+        start_after = self._startAfter_prefix if self._startAfter_prefix else self.prefix
+        self.logger.debug(
+            f"Making listreq with bucket:{self.bucket_name} prefix:{self.prefix} startAfter:{start_after}"
         )
+        list_dir = ListRequest(Bucket=self.bucket_name, Prefix=self.prefix, StartAfter=start_after)
         self._refresh_event_file_list_s3_mode(list_dir)

examples/profiler/sdk/smdebug-0.9.0b20200803-py2.py3-none-any.whl

@@ -0,0 +1,4 @@
+# Local
+from .metrics_histogram import MetricsHistogram
+from .step_histogram import StepHistogram
+from .training_job import TrainingJob

smdebug/profiler/algorithm_metrics_reader.py

@@ -12,71 +12,96 @@
 
 
 class MetricsHistogram:
-    def __init__(self, metrics_reader, select_metrics=None):
+    def __init__(self, metrics_reader):
 
         self.metrics_reader = metrics_reader
 
         # get timestamp of latest files
         self.last_timestamp = self.metrics_reader.get_timestamp_of_latest_available_file()
-        self.all_events = self.metrics_reader.get_events(0, self.last_timestamp)
-
-        # define the list of metrics to plot: per default cpu and gpu
-        self.select_metrics = ["cpu", "gpu"]
-        if select_metrics is not None:
-            self.select_metrics.extend(select_metrics)
-
-        self.preprocess_system_metrics()
-
-        self.create_plot()
-
-    def preprocess_system_metrics(self):
+        self.seen_system_metric_list = set()
+        self.select_metrics = []
+        self.sources = {}
+        self.target = None
+
+    """
+    @param starttime is starttime_since_epoch_in_micros. Default value 0, which means start
+    @param endtime is endtime_since_epoch_in_micros. Default value is  MetricsHistogram.last_timestamp , i.e., last_timestamp seen by system_metrics_reader
+    @param select_metrics is array of metrics to be selected, Default ["cpu", "gpu"]
+    """
+
+    def plot(self, starttime=0, endtime=None, select_metrics=[".*"]):
+        if endtime == None:
+            endtime = self.metrics_reader.get_timestamp_of_latest_available_file()
+        all_events = self.metrics_reader.get_events(starttime, endtime)
+        print(
+            f"Found {len(all_events)} system metrics events from timestamp_in_us:{starttime} to timestamp_in_us:{endtime}"
+        )
+        self.last_timestamp = endtime
+        self.select_metrics = select_metrics
+        self.system_metrics = self.preprocess_system_metrics(all_events=all_events)
+        self.create_plot(self.system_metrics)
+
+    def clear():
+        self.system_metrics = {}
+        self.sources = {}
 
+    def preprocess_system_metrics(self, all_events=[], system_metrics={}):
+        cpu_name = None
         # read all available system metric events and store them in dict
-        self.system_metrics = {}
-        for event in self.all_events:
-            if (
-                event.name not in self.system_metrics
-                and event.dimension is not "GPUMemoryUtilization"
-            ):
-                self.system_metrics[event.name] = []
-            self.system_metrics[event.name].append(event.value)
+        for event in all_events:
+            if event.name not in system_metrics:
+                system_metrics[event.name] = []
+                if cpu_name is None and event.dimension == "CPUUtilization":
+                    cpu_name = event.name
+                    print(cpu_name)
+            system_metrics[event.name].append(event.value)
 
         # total cpu utilization is not recorded in SM
-        self.cores = 0.0
-        cpu_total = np.zeros(len(self.system_metrics["cpu0"]))
-        for metric in self.system_metrics:
-            if "cpu" in metric:
-                self.cores += 1
-                cpu_total += self.system_metrics[metric]
+        if cpu_name is not None:
+            self.cores = 0.0
+            cpu_total = np.zeros(len(system_metrics[cpu_name]))
+            for metric in system_metrics:
+                # TODO should we do similar for gpu too
+                if "cpu" in metric and metric:
+                    if metric not in self.seen_system_metric_list:
+                        self.cores += 1
+                        self.seen_system_metric_list.add(metric)
 
-        self.system_metrics["cpu_total"] = cpu_total / self.cores
+                    cpu_total += system_metrics[metric]
 
-        # number of datapoints
-        self.width = self.system_metrics["cpu_total"].shape[0]
+            system_metrics["cpu_total"] = cpu_total / self.cores
 
         # add user defined metrics to the list
-        self.metrics = []
-        available_metrics = list(self.system_metrics.keys())
+        filtered_metrics = []
+        available_metrics = list(system_metrics.keys())
+        print(f"select metrics:{self.select_metrics}")
 
         for metric in self.select_metrics:
-            r = re.compile(".*" + metric)
-            self.metrics.extend(list(filter(r.match, available_metrics)))
-
-    def create_plot(self):
-
+            r = re.compile(r".*" + metric + r".*")
+            filtered_metrics.extend(list(filter(r.search, available_metrics)))
+        print(f"filtered_metrics:{filtered_metrics}")
+
+        # delete the keys which needs to be filtered out
+        for key in available_metrics:
+            if key not in filtered_metrics and "total" not in key:
+                del system_metrics[key]
+        return system_metrics
+
+    def _get_probs_binedges(self, values):
+        # create histogram bins
+        bins = np.arange(0, 100, 2)
+        probs, binedges = np.histogram(values, bins=bins)
+        bincenters = 0.5 * (binedges[1:] + binedges[:-1])
+        return probs, binedges
+
+    def create_plot(self, system_metrics={}):
+        metrics = list(system_metrics.keys())
         figures = []
-        self.sources = {}
 
         # create a histogram per metric
-        for index, metric in enumerate(self.metrics):
+        for index, metric in enumerate(metrics):
             p = figure(plot_height=250, plot_width=250)
-            values = self.system_metrics[metric]
-
-            # create histogram bins
-            bins = np.arange(0, 100, 2)
-            probs, binedges = np.histogram(values, bins=bins)
-            bincenters = 0.5 * (binedges[1:] + binedges[:-1])
-
+            probs, binedges = self._get_probs_binedges(system_metrics[metric])
             # set data
             source = ColumnDataSource(data=dict(top=probs, left=binedges[:-1], right=binedges[1:]))
             self.sources[metric] = source
@@ -93,47 +118,29 @@ def create_plot(self):
 
             # set plot
             p.y_range.start = 0
-            p.xaxis.axis_label = metric
+            p.xaxis.axis_label = metric + " util"
+            p.yaxis.axis_label = "Occurences"
             p.grid.grid_line_color = "white"
             figures.append(p)
 
         p = gridplot(figures, ncols=4)
         self.target = show(p, notebook_handle=True)
 
     def update_data(self, current_timestamp):
-
         # get all events from last to current timestamp
         events = self.metrics_reader.get_events(self.last_timestamp, current_timestamp)
         self.last_timestamp = current_timestamp
 
-        if len(events) > 0:
-            for event in events:
-                if event.name != None:
-                    self.system_metrics[event.name].append(event.value)
-
-            cpu_total = np.zeros(len(self.system_metrics["cpu0"]))
-
-            # iterate over available metrics
-            for metric in self.system_metrics:
+        self.system_metrics = self.preprocess_system_metrics(events, self.system_metrics)
 
-                # compute total cpu utilization
-                if "cpu" in metric and metric != "cpu_total":
-                    cpu_total += self.system_metrics[metric]
-
-            self.system_metrics["cpu_total"] = cpu_total / self.cores
-
-            # update histograms
-            for index, metric in enumerate(self.metrics):
-                values = self.system_metrics[metric]
-
-                # create new histogram bins
-                bins = np.arange(0, 100, 2)
-                probs, binedges = np.histogram(values, bins=bins)
-                bincenters = 0.5 * (binedges[1:] + binedges[:-1])
-
-                # update data
-                self.sources[metric].data["top"] = probs
-                self.sources[metric].data["left"] = binedges[:-1]
-                self.sources[metric].data["right"] = binedges[1:]
+        # update histograms
+        for index, metric in enumerate(self.system_metrics):
+            values = self.system_metrics[metric]
 
-            push_notebook()
+            # create new histogram bins
+            probs, binedges = self._get_probs_binedges(self.system_metrics[metric])
+            # update data
+            self.sources[metric].data["top"] = probs
+            self.sources[metric].data["left"] = binedges[:-1]
+            self.sources[metric].data["right"] = binedges[1:]
+        push_notebook()