From 5577ea9228547962c5dc50a2937e71b885c31291 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Sun, 5 May 2019 15:23:11 +0800 Subject: [PATCH 1/3] Update run_benchmark.py Support get the total running time when not build caffe with CAFFE_PER_LAYER_TIMINGS := 1 in train mode. --- scripts/run_benchmark.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index 8e90f08e1..ffe36df55 100755 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -273,6 +273,31 @@ def obtain_intelcaffe_log(self): logging.info('intelcaffe log: %s' % intelcaffe_log) return intelcaffe_log + def decode_sec(self, time_str): + from datetime import datetime + start_time = time_str.split()[2] + datetime_object = datetime.strptime(start_time, '%H:%M:%S.%f') + return datetime_object + + def obtain_total_time(self, result_file): + res = 0.0 + with open(result_file, 'r') as f: + delta_time_pattern = re.compile("Iteration 0") + first_line = "" + last_line = "" + for line in f.readlines(): + if first_line=="" and line.find("Iteration 0")>=0: + first_line = line + if line.find("Optimization Done.") >= 0: + last_line = line + + if first_line!="" and last_line!="": + start_time = self.decode_sec(first_line) + end_time = self.decode_sec(last_line) + res = (end_time- start_time).total_seconds() + + return res + def obtain_average_time(self): '''obtain average iteration time of training''' result_file = self.intelcaffe_log @@ -313,9 +338,15 @@ def obtain_average_time(self): if re.match(delta_time_pattern, line): delta_times.append(line.split()[-2]) if len(delta_times) == 0: - logging.exception("Error: check if you mark 'CAFFE_PER_LAYER_TIMINGS := 1' while building caffe; also ensure you're running at least 200 iterations for multinode trainings; or check if you're running intelcaffe failed, the logs can be found under: {}".format(result_file)) + logging.exception("Warn: check if you mark 'CAFFE_PER_LAYER_TIMINGS := 1' while building caffe; also ensure you're running at least 200 iterations for multinode trainings; or check if you're running intelcaffe failed, the logs can be found under: {}".format(result_file)) for delta_time in delta_times[start_iteration : start_iteration + iteration_num]: total_time += float(delta_time) + + if total_time == 0.0: + logging.info("Obtain total running time without CAFFE_PER_LAYER_TIMINGS := 1 in building caffe") + total_time = self.obtain_total_time(result_file) + iteration_num = 200 + average_time = total_time / iteration_num * 1000.0 logging.info("average time: {} ms".format(str(average_time))) return average_time @@ -343,6 +374,8 @@ def calculate_fps(self, model): '''calculate fps here''' self.batch_size = self.obtain_batch_size() self.average_time = self.obtain_average_time() + logging.info("batch size: %d, average time: %f" % \ + (self.batch_size, self.average_time)) speed = self.batch_size * 1000.0 / self.average_time self.speed = float(speed) total_speed = self.speed * int(self.inf_instances) From 3f49ec22f81d9feb5ebf07048df5ae0970f15559 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Sun, 5 May 2019 15:26:44 +0800 Subject: [PATCH 2/3] Update run_benchmark.py --- scripts/run_benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index ffe36df55..e644ef219 100755 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -6,6 +6,8 @@ import json import socket import logging +from datetime import datetime + class CaffeBenchmark(object): '''Used to do caffe benchmarking''' @@ -274,7 +276,6 @@ def obtain_intelcaffe_log(self): return intelcaffe_log def decode_sec(self, time_str): - from datetime import datetime start_time = time_str.split()[2] datetime_object = datetime.strptime(start_time, '%H:%M:%S.%f') return datetime_object From f7f6692d37808b3d5c9087da2ada0407aa7015a4 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Sun, 5 May 2019 15:27:34 +0800 Subject: [PATCH 3/3] Update run_benchmark.py --- scripts/run_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index e644ef219..d5f52cb25 100755 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -275,7 +275,7 @@ def obtain_intelcaffe_log(self): logging.info('intelcaffe log: %s' % intelcaffe_log) return intelcaffe_log - def decode_sec(self, time_str): + def decode_sec(self, time_str): start_time = time_str.split()[2] datetime_object = datetime.strptime(start_time, '%H:%M:%S.%f') return datetime_object