Skip to content

Commit 5153ac8

Browse files
mmglovemmglove
authored
fix profiler (#3323)
1 parent b060748 commit 5153ac8

File tree

2 files changed

+50
-24
lines changed

2 files changed

+50
-24
lines changed

‎paddlespeech/t2s/utils/profiler.py‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,17 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
import sys
1514

16-
import paddle
15+
import sys
16+
import paddle.profiler as profiler
1717

1818
# A global variable to record the number of calling times for profiler
1919
# functions. It is used to specify the tracing range of training steps.
2020
_profiler_step_id = 0
2121

2222
# A global variable to avoid parsing from string every time.
2323
_profiler_options = None
24-
24+
_prof = None
2525

2626
class ProfilerOptions(object):
2727
'''
@@ -31,6 +31,7 @@ class ProfilerOptions(object):
3131
"profile_path=model.profile"
3232
"batch_range=[50, 60]; profile_path=model.profile"
3333
"batch_range=[50, 60]; tracer_option=OpDetail; profile_path=model.profile"
34+
3435
ProfilerOptions supports following key-value pair:
3536
batch_range - a integer list, e.g. [100, 110].
3637
state - a string, the optional values are 'CPU', 'GPU' or 'All'.
@@ -52,7 +53,8 @@ def __init__(self, options_str):
5253
'sorted_key': 'total',
5354
'tracer_option': 'Default',
5455
'profile_path': '/tmp/profile',
55-
'exit_on_finished': True
56+
'exit_on_finished': True,
57+
'timer_only': True
5658
}
5759
self._parse_from_string(options_str)
5860

@@ -71,6 +73,8 @@ def _parse_from_string(self, options_str):
7173
'state', 'sorted_key', 'tracer_option', 'profile_path'
7274
]:
7375
self._options[key] = value
76+
elif key == 'timer_only':
77+
self._options[key] = value
7478

7579
def __getitem__(self, name):
7680
if self._options.get(name, None) is None:
@@ -84,26 +88,40 @@ def add_profiler_step(options_str=None):
8488
Enable the operator-level timing using PaddlePaddle's profiler.
8589
The profiler uses a independent variable to count the profiler steps.
8690
One call of this function is treated as a profiler step.
87-
8891
Args:
8992
profiler_options - a string to initialize the ProfilerOptions.
9093
Default is None, and the profiler is disabled.
9194
'''
9295
if options_str is None:
9396
return
9497

98+
global _prof
9599
global _profiler_step_id
96100
global _profiler_options
97101

98102
if _profiler_options is None:
99103
_profiler_options = ProfilerOptions(options_str)
100-
101-
if _profiler_step_id == _profiler_options['batch_range'][0]:
102-
paddle.utils.profiler.start_profiler(_profiler_options['state'],
103-
_profiler_options['tracer_option'])
104-
elif _profiler_step_id == _profiler_options['batch_range'][1]:
105-
paddle.utils.profiler.stop_profiler(_profiler_options['sorted_key'],
106-
_profiler_options['profile_path'])
104+
# profile : https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/performance_improving/profiling_model.html#chakanxingnengshujudetongjibiaodan
105+
# timer_only = True only the model's throughput and time overhead are displayed
106+
# timer_only = False calling summary can print a statistical form that presents performance data from different perspectives.
107+
# timer_only = False the output Timeline information can be found in the profiler_log directory
108+
if _prof is None:
109+
_timer_only = str(_profiler_options['timer_only']) == str(True)
110+
_prof = profiler.Profiler(
111+
scheduler = (_profiler_options['batch_range'][0], _profiler_options['batch_range'][1]),
112+
on_trace_ready = profiler.export_chrome_tracing('./profiler_log'),
113+
timer_only = _timer_only)
114+
_prof.start()
115+
else:
116+
_prof.step()
117+
118+
if _profiler_step_id == _profiler_options['batch_range'][1]:
119+
_prof.stop()
120+
_prof.summary(
121+
op_detail=True,
122+
thread_sep=False,
123+
time_unit='ms')
124+
_prof = None
107125
if _profiler_options['exit_on_finished']:
108126
sys.exit(0)
109127

‎tests/test_tipc/benchmark_train.sh‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ repo_name=$(get_repo_name )
110110
SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log
111111
mkdir -p "${SAVE_LOG}/benchmark_log/"
112112
status_log="${SAVE_LOG}/benchmark_log/results.log"
113+
# get benchmark profiling params : PROFILING_TIMER_ONLY=no|True|False
114+
PROFILING_TIMER_ONLY=${PROFILING_TIMER_ONLY:-"True"}
113115

114116
# The number of lines in which train params can be replaced.
115117
line_python=3
@@ -166,19 +168,25 @@ for batch_size in ${batch_size_list[*]}; do
166168
gpu_id=$(set_gpu_id $device_num)
167169

168170
if [ ${#gpu_id} -le 1 ];then
169-
log_path="$SAVE_LOG/profiling_log"
170-
mkdir -p $log_path
171-
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_profiling"
172171
func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id
173-
# set profile_option params
174-
tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`
175-
176-
# run test_train_inference_python.sh
177-
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
178-
echo $cmd
179-
eval $cmd
180-
eval "cat ${log_path}/${log_name}"
181-
172+
if [[ ${PROFILING_TIMER_ONLY} != "no" ]];then
173+
echo "run profile"
174+
# The default value of profile_option's timer_only parameter is True
175+
if [[ ${PROFILING_TIMER_ONLY} = "False" ]];then
176+
profile_option="${profile_option};timer_only=False"
177+
fi
178+
log_path="$SAVE_LOG/profiling_log"
179+
mkdir -p $log_path
180+
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_${to_static}profiling"
181+
# set profile_option params
182+
tmp=`sed -i "${line_profile}s/.*/\"${profile_option}\"/" "${FILENAME}"`
183+
# run test_train_inference_python.sh
184+
cmd="timeout 5m bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
185+
echo $cmd
186+
eval ${cmd}
187+
eval "cat ${log_path}/${log_name}"
188+
fi
189+
echo "run without profile"
182190
# without profile
183191
log_path="$SAVE_LOG/train_log"
184192
speed_log_path="$SAVE_LOG/index"

0 commit comments

Comments
 (0)