# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """The integrator for integrating parsed profiling files.""" import csv import json import os import stat from decimal import Decimal from mindspore import log as logger from mindspore.context import get_auto_parallel_context from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException, \ ProfilerFileNotFoundException, ProfilerRawFileException, ProfilerParamValueErrorException from mindspore.profiler.common.util import query_latest_trace_time_file, to_int, to_millisecond from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path from mindspore.profiler.parser.container import TimelineContainer SIZE_LIMIT_DEFAULT = 20 * 1024 * 1024 # 20MB class Integrator: """ The integrator for integrating parsed profiling files. Args: profiling_dir (str): The directory where the parsed profiling files are located. device_id (str): The device ID. """ _file_name_aicore_detail_time = 'output_op_compute_time_{}.txt' _file_name_aicpu_time = 'output_data_preprocess_aicpu_{}.txt' _file_name_framework = 'framework_raw_{}.csv' _header_aicore_type = ['op_type', 'execution_time', 'execution_frequency', 'percent'] _header_aicore_detail = ['full_op_name', 'execution_time'] _header_aicpu = ['serial_number', 'op_type', 'total_time', 'dispatch_time', 'execution_time', 'run_start', 'run_end'] _file_name_aicore_type_time = 'aicore_intermediate_{}_type.csv' _file_name_aicore_detail_info = 'aicore_intermediate_{}_detail.csv' _col_names_detail = ['op_name', 'op_type', 'avg_execution_time', 'subgraph', 'full_op_name', 'op_info'] _none_filter_condition_key = ['is_display_detail', 'is_display_full_op_name'] _none_sort_col_names = ['op_info'] _aicore_data = [] _aicore_detail_data = [] _aicore_trace_data = [] def __init__(self, profiling_dir, device_id): self._profiling_dir = profiling_dir self._device_id = device_id self._op_time_cache = {} self._total_time = Decimal('0.0') self._column = "" self._result = [] def integrate(self): """Integrate the parsed profiling files.""" self._parse_aicore_detail_time() self._parse_aicore_type_time() self._parse_aicpu_time() def get_aicore_data(self): self._aicore_data_load() return self._aicore_data def get_aicore_detail_data(self): self._aicore_detail_data_load() return self._aicore_detail_data def get_aicore_trace_data(self): self._aicore_trace_data_load() return self._aicore_trace_data def query_for_all_reduce(self): return self._query_for_all_reduce() def query_and_sort_by_op_type(self, filter_condition, op_type_order): return self._query_and_sort_by_op_type(filter_condition, op_type_order) def _parse_aicore_type_time(self): """Parse the parsed AICORE operator type file.""" framework_file = os.path.join( self._profiling_dir, self._file_name_framework.format(self._device_id) ) framework_file = validate_and_normalize_path(framework_file) if not os.path.isfile(framework_file): return op_name_type_cache = {} with open(framework_file, 'r') as src_file: csv_reader = csv.reader(src_file) _ = next(csv_reader) for row in csv_reader: op_name_type_cache[row[3]] = row[5] op_type_time_cache = {} for full_op_name, op_time in self._op_time_cache.items(): op_type = op_name_type_cache.get(full_op_name) if op_type_time_cache.get(op_type) is None: op_type_time_cache[op_type] = [op_time, 1] else: op_type_time_cache[op_type][0] += op_time op_type_time_cache[op_type][1] += 1 if self._total_time == 0: raise ValueError("The total time of operations can not be 0.") op_type_file_name = 'aicore_intermediate_' + self._device_id + '_type.csv' op_type_file_path = os.path.join(self._profiling_dir, op_type_file_name) with open(op_type_file_path, 'w') as type_file: csv_writer = csv.writer(type_file) csv_writer.writerow(self._header_aicore_type) for op_type, op_type_time_info in op_type_time_cache.items(): type_info = [ op_type, op_type_time_info[0], op_type_time_info[1], round((op_type_time_info[0] / self._total_time) * 100, 2) ] csv_writer.writerow(type_info) def _parse_aicore_detail_time(self): """Parse the parsed AICORE operator time file.""" aicore_detail_file = os.path.join( self._profiling_dir, self._file_name_aicore_detail_time.format(self._device_id) ) aicore_detail_file = validate_and_normalize_path(aicore_detail_file) if not os.path.isfile(aicore_detail_file): return op_detail_file_name = 'aicore_intermediate_' + self._device_id + '_detail.csv' op_detail_file_path = os.path.join( self._profiling_dir, op_detail_file_name ) with open(aicore_detail_file, 'r') as src_file: row = src_file.readline() if row.startswith('op_name'): _ = src_file.readline() elif row.startswith('====='): _ = src_file.readline() _ = src_file.readline() else: return with open(op_detail_file_path, 'w') as detail_file: csv_writer = csv.writer(detail_file) csv_writer.writerow(self._header_aicore_detail) while True: row = src_file.readline() if not row: break op_infos = row.split() if op_infos[0] == 'total': self._total_time = Decimal(op_infos[2]) continue self._op_time_cache[op_infos[0]] = Decimal(op_infos[1]) csv_writer.writerow([op_infos[0], op_infos[1]]) def _parse_aicpu_time(self): """Parse the parsed AICPU operator time file.""" aicpu_file = os.path.join( self._profiling_dir, self._file_name_aicpu_time.format(self._device_id) ) aicpu_file = validate_and_normalize_path(aicpu_file) if not os.path.isfile(aicpu_file): return save_file_name = 'aicpu_intermediate_' + self._device_id + '.csv' save_file_path = os.path.join(self._profiling_dir, save_file_name) with open(aicpu_file, 'r') as src_file: row = src_file.readline() if not row.startswith('serial_number'): return with open(save_file_path, 'w') as save_file: csv_writer = csv.writer(save_file) csv_writer.writerow(self._header_aicpu) while True: row = src_file.readline() if not row: break infos = row.split() if infos[0] == 'AI': continue csv_writer.writerow(infos) def _aicore_data_load(self): """Load data according to the parsed AICORE operator types file.""" op_type_file_path = os.path.join( self._profiling_dir, self._file_name_aicore_type_time.format(self._device_id) ) op_type_file_path = validate_and_normalize_path(op_type_file_path) if not os.path.isfile(op_type_file_path): logger.warning('The file <%s> does not exist.', op_type_file_path) return with open(op_type_file_path, 'r') as file: csv_reader = csv.reader(file) _ = next(csv_reader) for info in csv_reader: self._aicore_data.append([info[0], float(info[1]), int(info[2]), float(info[3])]) def _aicore_detail_data_load(self): """Load data according to the parsed AICORE operator file.""" op_detail_file_path = os.path.join( self._profiling_dir, self._file_name_aicore_detail_info.format(self._device_id) ) framework_file_path = os.path.join( self._profiling_dir, self._file_name_framework.format(self._device_id) ) op_detail_file_path = validate_and_normalize_path(op_detail_file_path) framework_file_path = validate_and_normalize_path(framework_file_path) if not os.path.isfile(op_detail_file_path): logger.warning('The file <%s> does not exist.', op_detail_file_path) return if not os.path.isfile(framework_file_path): logger.warning('The file <%s> does not exist.', framework_file_path) return framework_infos = dict() with open(framework_file_path, 'r') as file: csv_reader = csv.reader(file) _ = next(csv_reader) for info in csv_reader: framework_infos[info[3]] = [ info[3], info[4], info[5], info[6], json.loads(info[7]) if info[7] else None] with open(op_detail_file_path, 'r') as file: csv_reader = csv.reader(file) _ = next(csv_reader) for info in csv_reader: framework_info = framework_infos.get(info[0]) self._aicore_detail_data.append( [ framework_info[1], framework_info[2], float(info[1]), framework_info[3], framework_info[0], framework_info[4] ] ) del framework_infos def _aicore_trace_data_load(self): """Load data according to the parsed AICORE operator types file.""" file_path = query_latest_trace_time_file(self._profiling_dir, int(self._device_id)) if not file_path: logger.critical("Failed to find parsed trace time file.") raise ProfilerFileNotFoundException('parsed step trace time file') file_path = validate_and_normalize_path(file_path) with open(file_path, 'r') as handle: csv_reader = csv.reader(handle) self._column = next(csv_reader) self._aicore_trace_data = list(csv_reader) self._size = len(self._aicore_trace_data) - 1 self._load_point_info() def _load_point_info(self): """Load point info.""" file_path = os.path.join(self._profiling_dir, 'step_trace_point_info.json') file_path = validate_and_normalize_path(file_path) if os.path.isfile(file_path): with open(file_path, 'r', encoding='utf-8') as file: try: self._point_info = json.load(file) except (json.JSONDecodeError, TypeError) as err: logger.warning(err) raise ProfilerRawFileException('Fail to parse point info file.') def _query_for_all_reduce(self): """ Query for all reduce info. Returns: list[dict], reduce information. Each item is the reduce info for one step. The reduce info is format like: {stream_id: List[Tuple(start_point, end_point, duration, field_name)]}. """ self._aicore_trace_data_load() reduce_infos = [] for row_info in self._aicore_trace_data[:-1]: row_info_dict = self._get_info_dict_from_row_data(row_info, 'systime') reduce_info = self._sort_reduce_by_time(row_info_dict) if reduce_info: reduce_infos.extend(reduce_info) reduce_infos.sort(key=lambda x: float(x[2])) return reduce_infos def _get_info_dict_from_row_data(self, row_info, time_type): """ Get step info in dict format. Args: row_info (list[str]): Step info, the value is corresponding to `__column__`. time_type (str): The value type. `systime` keeps the original value. `realtime` transforms the value in millisecond. Default: `realtime`. Returns: dict, step trace information. The key is in `__column__`. """ row_info_dict = {} for key, value in zip(self._column, row_info): if key == 'step_num': continue value = to_int(value, key) row_info_dict[key] = to_millisecond(value) if time_type == 'realtime' else value return row_info_dict def _sort_reduce_by_time(self, row_info_dict): """ Sort reduce info by time. Args: row_info_dict (dict): Step trace information. Returns: list, including the all reduce info sorted by start time only. [ [reduce_field, stream_id, reduce_start, reduce_duration], [...], [...] ] """ factor = 1e5 # convert time unit from 10ns to 1ms reduce_pid = 10000 reduce_info = [] reduce_fields = [field_name for field_name in self._column if field_name.startswith('stream_') and not field_name.endswith('point')] for reduce_field in reduce_fields: reduce_start = row_info_dict.get(reduce_field + '_start_point') reduce_start = reduce_start / factor \ if reduce_start else 0 reduce_duration = row_info_dict.get(reduce_field) reduce_duration = reduce_duration / factor if reduce_duration else 0 if not (reduce_start and reduce_duration): logger.info("Reduce event missing value.") continue cur_stream_id = reduce_field.split('_', 2)[1] reduce_meta = [reduce_field, int(cur_stream_id), reduce_start, reduce_duration, reduce_pid] reduce_info.append(reduce_meta) return reduce_info def _query_and_sort_by_op_type(self, filter_condition, op_type_order: list): """ Query the AICORE operator detail information by `filter_condition`, and sort by `op_type_order` and execution time. Args: filter_condition (dict): The filter condition. op_type_order (list[str]): The name of the operator type in order. Returns: dict, The results are filtered and sorted. """ self._aicore_detail_data_load() if filter_condition is None: filter_condition = {} self._filter(filter_condition) type_detail_cache = {} for detail_info in self._result: op_type = detail_info[1] if op_type not in op_type_order: continue infos = type_detail_cache.get(op_type) if infos: infos.append(detail_info) else: type_detail_cache[op_type] = [detail_info] result = [] for op_type in op_type_order: detail_infos = type_detail_cache.get(op_type) if detail_infos is None: continue detail_infos.sort(key=lambda item: item[2], reverse=True) result.extend(detail_infos) return { 'col_name_detail': self._display_col_names_detail, 'object': result } def _filter(self, filter_condition): """ Filter the profiling data according to the filter condition. Args: filter_condition (dict): The filter condition. """ def _inner_filter(item: list): return self._default_filter(item, filter_condition) def _inner_map(item: list): inner_item = item[0:4] if is_display_full_op_name: inner_item.append(item[4]) if is_display_detail: inner_item.append(item[5]) return inner_item is_display_detail = filter_condition.get('is_display_detail', True) is_display_full_op_name = filter_condition.get( 'is_display_full_op_name', True ) self._set_display_col_name(is_display_detail, is_display_full_op_name) if is_display_detail and is_display_full_op_name: self._result = list(filter(_inner_filter, self._aicore_detail_data)) else: self._result = list( map(_inner_map, filter(_inner_filter, self._aicore_detail_data)) ) def _default_filter(self, item, condition): """ The default filter method. Args: item (list[Union[str, float, int]]): A piece of data to be filtered. condition (dict): The filter condition. Returns: bool, `True` if the item is satisfied. """ for condition_key, condition_value in condition.items(): if condition_key in self._none_filter_condition_key: continue if condition_key in self._col_names_detail: index = self._col_names_detail.index(condition_key) actual_value = item[index] for exp_key, exp_value in condition_value.items(): if not self._is_match_condition( exp_key, exp_value, actual_value): return False return True @staticmethod def _is_match_condition(exp_key, exp_value, actual_value): """ Check whether the actual value meets the expect condition. Args: exp_key (str): Expect key of the condition. exp_value (str): Expect value. actual_value (str): Actual value. Returns: bool, `True` if the actual meets the expect condition, else `False`. """ if exp_key == 'in': if actual_value not in exp_value: return False elif exp_key == 'not_in': if actual_value in exp_value: return False elif exp_key == 'partial_match_str_in': for partial_match_str in exp_value: if partial_match_str in actual_value: return True return False else: return False return True def _set_display_col_name(self, is_display_detail, is_display_full_op_name): """ Set the display column name according to the filter condition. Args: is_display_detail (bool): Whether to display the detailed operator information. is_display_full_op_name (bool): Whether to display the operator full name. """ self._display_col_names_detail = self._col_names_detail[0:4] if is_display_full_op_name: self._display_col_names_detail.append(self._col_names_detail[4]) if is_display_detail: self._display_col_names_detail.append(self._col_names_detail[5]) class BaseTimelineGenerator: """ Analyse timeline data from file. """ __col_names__ = ['op_name', 'stream_id', 'start_time', 'duration'] _output_timeline_data_file_path = 'output_timeline_data_{}.txt' _timeline_meta = [] _format_meta_data_list = [] _thread_processed_list = [] # AI Core Op pid is device_id _AI_CPU_PID = 9000 _COMMUNICATION_OP_PID = 10000 _HOST_CPU_PID = 11000 _OP_OVERLAP_PID = 12000 _MERGED_COMPUTATION_TID = 7999 _PURE_COMMUNICATION_TID = 8000 _MERGED_COMMUNICATION_TID = 8001 _FREE_TIME_TID = 8002 _STEPS_TID = 100000 _SCOPE_NAME_TID = 100001 _GPU_OP_TID = 100002 _HOST_CPU_OP_TID = 100003 _map_tid_name_to_int = { "Steps": (-4, _STEPS_TID), "Scope Name": (-3, _SCOPE_NAME_TID), "GpuOps": (-2, _GPU_OP_TID), "HostCpuOps": (-1, _HOST_CPU_OP_TID) } _timeline_summary = { 'total_time': 0, 'num_of_streams': 0, 'num_of_ops': 0, 'op_exe_times': 0, 'max_scope_name_num': 0, } _op_name_idx, _tid_idx, _start_time_idx, _duration_idx = 0, 1, 2, 3 _max_scope_name_num = 0 _host_cpu_op_label = 'HostCpuOps' def get_thread_label_name(self): """Get process and thread config.""" return [ {"name": "process_labels", "ph": "M", "pid": self._device_id, "args": {"labels": "AI Core Op"}}, {"name": "process_labels", "ph": "M", "pid": self._AI_CPU_PID, "args": {"labels": "AI CPU Op"}}, {"name": "process_labels", "ph": "M", "pid": self._COMMUNICATION_OP_PID, "args": {"labels": "Communication Op"}}, {"name": "process_labels", "ph": "M", "pid": self._HOST_CPU_PID, "args": {"labels": "Host CPU Op"}}, {"name": "process_labels", "ph": "M", "pid": self._OP_OVERLAP_PID, "args": {"labels": "Op Overlap Analyse"}}, {"name": "process_sort_index", "ph": "M", "pid": self._device_id, "args": {"sort_index": 0}}, {"name": "process_sort_index", "ph": "M", "pid": self._AI_CPU_PID, "args": {"sort_index": 10}}, {"name": "process_sort_index", "ph": "M", "pid": self._COMMUNICATION_OP_PID, "args": {"sort_index": 20}}, {"name": "process_sort_index", "ph": "M", "pid": self._HOST_CPU_PID, "args": {"sort_index": 30}}, {"name": "process_sort_index", "ph": "M", "pid": self._OP_OVERLAP_PID, "args": {"sort_index": 40}}, {"name": "thread_name", "ph": "M", "pid": self._HOST_CPU_PID, "tid": self._HOST_CPU_OP_TID, "args": {"name": "Host CPU Op"}}, {"name": "thread_name", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._MERGED_COMPUTATION_TID, "args": {"name": "Merged Computation Op"}}, {"name": "thread_name", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._PURE_COMMUNICATION_TID, "args": {"name": "Pure Communication Op"}}, {"name": "thread_name", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._MERGED_COMMUNICATION_TID, "args": {"name": "Merged Communication Op"}}, {"name": "thread_name", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._FREE_TIME_TID, "args": {"name": "Free Time"}}, {"name": "thread_sort_index", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._MERGED_COMPUTATION_TID, "args": {"sort_index": self._MERGED_COMPUTATION_TID}}, {"name": "thread_sort_index", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._PURE_COMMUNICATION_TID, "args": {"sort_index": self._PURE_COMMUNICATION_TID}}, {"name": "thread_sort_index", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._MERGED_COMMUNICATION_TID, "args": {"sort_index": self._MERGED_COMMUNICATION_TID}}, {"name": "thread_sort_index", "ph": "M", "pid": self._OP_OVERLAP_PID, "tid": self._FREE_TIME_TID, "args": {"sort_index": self._FREE_TIME_TID}} ] def write_timeline(self, size_limit=SIZE_LIMIT_DEFAULT): """Load data according to the parsed profiling files.""" # Write timeline to file. logger.info('Writing timeline file...') self.write_timeline_to_json_by_limitation(size_limit) logger.info('Finished file writing!') def write_timeline_to_json_by_limitation(self, size_limit): """Write timeline to json by limitation.""" display_file_path = os.path.join( self._profiling_dir, self._display_filename ) display_file_path = validate_and_normalize_path(display_file_path) try: with open(display_file_path, 'w') as json_file: json_file.write('[') for _, item in enumerate(self._timeline_meta): json.dump(item, json_file) if "scope_level" in item.keys(): self._max_scope_name_num = max( self._max_scope_name_num, item["scope_level"] + 1) file_size = os.path.getsize(display_file_path) json_file.write(',') if file_size > size_limit: break label_name_json = json.dumps(self.get_thread_label_name()) label_name_json = label_name_json.lstrip('[') json_file.write(label_name_json) os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.critical('Error occurred when write timeline display file: %s', err) raise ProfilerIOException() def write_timeline_summary(self): """Write timeline summary to json.""" timeline_summary_file_path = os.path.join( self._profiling_dir, self._timeline_summary_filename ) timeline_summary_file_path = validate_and_normalize_path(timeline_summary_file_path) try: with open(timeline_summary_file_path, 'w') as json_file: json.dump(self._timeline_summary, json_file) os.chmod(timeline_summary_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.critical('Error occurred when write timeline summary file: %s', err) raise ProfilerIOException() @staticmethod def _update_num_of_streams(timeline, stream_count_dict): """Update number of streams.""" stream_id = timeline[1] if stream_id in ["Steps", "Scope Name"]: return if stream_id not in stream_count_dict.keys(): stream_count_dict[stream_id] = 1 else: stream_count_dict[stream_id] += 1 def _update_format_meta_data(self, timeline_dict): """Update format meta data which control the display arrange and map the thread name.""" thread_name_meta_data = { "name": "thread_name", "pid": int(self._device_id), "tid": 100000, "ts": 0, "ph": "M", "cat": "__metadata", "args": { "name": "Steps" } } tid_name = timeline_dict['tid'] sort_index = 0 if tid_name in self._map_tid_name_to_int.keys(): sort_index, tid = self._map_tid_name_to_int[tid_name] elif tid_name.startswith("Stream"): tid = int(tid_name.split("#")[-1]) sort_index = tid else: return if tid_name == self._host_cpu_op_label: thread_name_meta_data['pid'] = self._HOST_CPU_PID thread_name_meta_data["tid"] = tid thread_name_meta_data["args"]["name"] = tid_name thread_sort_meta_data = thread_name_meta_data.copy() thread_sort_meta_data['name'] = "thread_sort_index" thread_sort_meta_data["args"] = {"sort_index": sort_index} timeline_dict["tid"] = tid if tid_name in self._thread_processed_list: return self._thread_processed_list.append(tid_name) self._format_meta_data_list.append(thread_name_meta_data) self._format_meta_data_list.append(thread_sort_meta_data) def _get_max_scope_name_num(self, timeline_list): """Get the max number of scope level from all operator.""" max_scope_name_num = 0 for time_item in timeline_list: cur_scope_name_num = len(time_item[self._op_name_idx].split('/')) - 1 max_scope_name_num = max(cur_scope_name_num, max_scope_name_num) return max_scope_name_num def _get_scope_name_time_list(self, timeline_list, subgraph, factor_start_time_to_duration=1): """Produce the timeline of hierarchical scope name.""" # the key of scope_name_start_duration_dict is scope name, the value is a dict which store the # start and end index of time_item in timeline_list. scope_name_start_duration_dict = {} scope_name_time_list = [] op_full_name_idx, scope_name_idx, invalid_idx = 0, 0, -1 tid = "Scope Name" for idx, time_item in enumerate(timeline_list): scope_name_list = time_item[op_full_name_idx].split('/')[:-1] # skip Default/InitDataSetQueue operator. if time_item[op_full_name_idx].startswith("Default/InitDataSetQueue"): scope_name_list = [] # process scope name of subgraph(Default/Gradients/recompute_Default) only. if scope_name_list and scope_name_list[0] != subgraph: scope_name_list = [] # add the level of scope name, used to distinguish the same name at different scope level. scope_name_list = [f"{scope_level}-{scope_name}" for scope_level, scope_name in enumerate(scope_name_list)] # update the start and end index of time_item according to current scope_name for scope_name in scope_name_list: init_start_end_idx_dict = {'start_item_idx': idx, 'end_item_idx': idx} if scope_name not in scope_name_start_duration_dict: scope_name_start_duration_dict[scope_name] = init_start_end_idx_dict if scope_name_start_duration_dict[scope_name]['start_item_idx'] == invalid_idx: scope_name_start_duration_dict[scope_name] = init_start_end_idx_dict else: scope_name_start_duration_dict[scope_name]['end_item_idx'] = idx # if the key(scope name) in scope_name_start_duration_dict does not appear in scope_name_list, # it means this key(scope name) is end and it is append to scope_name_time_list. for key, val in scope_name_start_duration_dict.items(): if val['start_item_idx'] == invalid_idx: continue if (key not in scope_name_list) \ or idx == (len(timeline_list) - 1) \ or time_item[op_full_name_idx] == self._step_end_op_name: start_time = timeline_list[val['start_item_idx']][self._start_time_idx] duration = (float(timeline_list[val['end_item_idx']][self._start_time_idx]) - float(start_time)) * \ factor_start_time_to_duration + float(timeline_list[val['end_item_idx']][self._duration_idx]) scope_name_time_item = [key, tid, start_time, duration] scope_name_time_list.append(scope_name_time_item) scope_name_start_duration_dict[key]['start_item_idx'] = invalid_idx # x[scope_name_idx] is a scope name like "0-Default". # if two element in scope_name_time_list have the same start time, # the previous element in list will displayed at the higher line in UI page. scope_name_time_list.sort( key=lambda x: (float(x[self._start_time_idx]), int(x[scope_name_idx].split('-')[0])) ) return scope_name_time_list def _set_step_start_and_end_op_name(self, timeline_list): """Set the start and end operator full name of each step.""" if not timeline_list: return start_op_idx = 0 if timeline_list[0][self._op_name_idx].startswith("Default/InitDataSetQueue"): start_op_idx = 1 self._step_start_op_name = timeline_list[start_op_idx][self._op_name_idx] self._step_end_op_name = self._step_start_op_name if len(timeline_list) > (start_op_idx + 1): for time_item in timeline_list[start_op_idx + 1:]: if time_item[self._op_name_idx] != self._step_start_op_name: self._step_end_op_name = time_item[self._op_name_idx] else: break def _get_step_time_list(self, timeline_list, factor_start_time_to_duration=1): """Produce the time of each step.""" # Record the time of each step. step_time_list = [] step_num = 1 tid = "Steps" cur_step_start_time, cur_step_duration_time = 0, 0 for time_item in timeline_list: if time_item[self._op_name_idx] == self._step_start_op_name: cur_step_start_time = time_item[self._start_time_idx] if time_item[self._op_name_idx] == self._step_end_op_name: cur_step_duration_time = (float(time_item[self._start_time_idx]) - float(cur_step_start_time)) * \ factor_start_time_to_duration + float(time_item[self._duration_idx]) step_time_item = [str(step_num), tid, float(cur_step_start_time), cur_step_duration_time] step_time_list.append(step_time_item) step_num += 1 return step_time_list class GpuTimelineGenerator(BaseTimelineGenerator): """Generate gpu Timeline data from file.""" _display_filename = 'gpu_timeline_display_{}.json' _timeline_summary_filename = 'gpu_timeline_summary_{}.json' _output_op_execute_time_file_path = "gpu_op_execute_timestamp_{}.txt" _output_activity_execute_time_file_path = "activity_execute_timestamp_{}.txt" _output_gpu_activity_info_file_path = "gpu_activity_data_{}.csv" _step_trace_original_filename = 'step_trace_profiling_{}.txt' _activity_keys_list = [] def __init__(self, profiling_dir, device_id): self._profiling_dir = profiling_dir self._device_id = device_id self._timeline_meta = [] self._display_filename = self._display_filename.format(device_id) self._timeline_summary_filename = self._timeline_summary_filename.format(device_id) def _get_and_validate_path(self, file_name): """Generate op or activity file path from file name, and validate this path.""" file_path = os.path.join( self._profiling_dir, file_name.format(self._device_id) ) file_path = validate_and_normalize_path(file_path) if not os.path.exists(file_path): logger.critical(f"Failed to find parsed timeline file {file_path}.") raise ProfilerFileNotFoundException('parsed timeline file') return file_path def _parse_timeline_data(self, timeline, min_cycle_counter): """Parse timeline data.""" # factor to convert the time unit of start_time(ts) from 1ns to 1us for timeline display factor = 1000 op_meta = TimelineContainer(timeline) timeline_dict = {} timeline_dict['name'] = op_meta.op_name.split('/')[-1] timeline_dict['ph'] = 'X' timeline_dict['tid'] = op_meta.stream_id timeline_dict['ts'] = (op_meta.start_time - min_cycle_counter) / factor dur = op_meta.duration timeline_dict['dur'] = dur timeline_dict['pid'] = int(self._device_id) if op_meta.stream_id == "Scope Name": # remove the level of scope name which has a format like "0-conv2-Conv2d". timeline_dict['name'] = "-".join(op_meta.op_name.split('-')[1:]) timeline_dict['scope_level'] = int(op_meta.op_name.split('-')[0]) elif op_meta.stream_id == self._host_cpu_op_label: timeline_dict['pid'] = self._HOST_CPU_PID if len(timeline) > 4: # len(timeline) > 4 refers to activity data, else op data. # Add args for activity data args_dict = {} for ix, value in enumerate(timeline[4:]): args_dict[self._activity_keys_list[ix]] = value timeline_dict['args'] = args_dict timeline_dict['tid'] = f"Stream #{timeline_dict['tid']}" elif op_meta.stream_id not in ["Scope Name", "Steps"]: # Update total time of operator execution. self._timeline_summary['total_time'] += dur / factor self._timeline_summary['op_exe_times'] += 1 self._update_format_meta_data(timeline_dict) self._timeline_meta.append(timeline_dict) def _load_timeline_data(self): """Load timeline data from file.""" op_file_path = self._get_and_validate_path( self._output_op_execute_time_file_path) activity_file_path = self._get_and_validate_path( self._output_activity_execute_time_file_path) activity_args_file_path = self._get_and_validate_path( self._output_gpu_activity_info_file_path) timeline_list = self._load_op_data(op_file_path) # Add host cpu op timeline. cpu_timeline_generator = CpuTimelineGenerator(self._profiling_dir, self._device_id) cpu_timeline_list = cpu_timeline_generator.load_cpu_op_data() if cpu_timeline_list: self._clock_synchronize_to_gpu(cpu_timeline_list) timeline_list.extend(cpu_timeline_list) timeline_list.sort(key=lambda x: float(x[2])) self._max_scope_name_num = self._get_max_scope_name_num(timeline_list) self._timeline_summary['max_scope_name_num'] = self._max_scope_name_num # Generate step time. factor_start_time_uint_to_duration = 1e-3 self._set_step_start_and_end_op_name(timeline_list) # Fit gpu kernel async launch solution. if self.is_gpu_kernel_async_launch(): step_time_list = self._get_step_time_list_from_step_trace() else: step_time_list = self._get_step_time_list(timeline_list, factor_start_time_uint_to_duration) # Add Scope Name. default_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Default", factor_start_time_uint_to_duration) gradient_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Gradients", factor_start_time_uint_to_duration) recompute_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "recompute_Default", factor_start_time_uint_to_duration) timeline_list.extend(default_scope_name_time_list) timeline_list.extend(gradient_scope_name_time_list) timeline_list.extend(recompute_scope_name_time_list) timeline_list.extend(step_time_list) timeline_list.sort(key=lambda x: (float(x[self._start_time_idx]), x[self._tid_idx])) # Add cuda activity timeline. activity_timeline_list = self._load_activity_data(activity_file_path, activity_args_file_path) timeline_list.extend(activity_timeline_list) timeline_list.sort(key=lambda x: float(x[2])) return timeline_list def _clock_synchronize_to_gpu(self, timeline_list): """Synchronize the timestamp from device to host.""" start_time_file_path = os.path.join(self._profiling_dir, f"start_time_{self._device_id}.txt") try: with open(start_time_file_path) as f_obj: lines = f_obj.readlines() # lines[0] stores the host monotonic time of start training. host_monotonic_start_time = int(lines[0].strip().split(':')[-1]) # lines[1] stores the gpu time of start training. gpu_start_time = int(lines[1].strip().split(':')[-1]) except (IOError, OSError) as err: logger.critical(f'Error occurred when read {start_time_file_path}: {err}') raise ProfilerIOException() time_diff = gpu_start_time - host_monotonic_start_time for idx, time_item in enumerate(timeline_list): timeline_list[idx][self._start_time_idx] = int(time_item[self._start_time_idx]) + time_diff def _load_op_data(self, op_file_path): """Load operator data from file""" op_timeline_list = [] try: with open(op_file_path, 'r') as f_obj: for line in f_obj: self._timeline_summary['num_of_ops'] += 1 op_list = line.strip('\n').strip().split(';') time_arr = op_list[-1] time_arr = time_arr.split(" ") for time in time_arr: time = time.split(",") line_list = op_list[:2] + time op_timeline_list.append(line_list) except (IOError, OSError) as err: logger.critical('Error occurred when load operator timeline data intermediate file: %s', err) raise ProfilerIOException() return op_timeline_list def _load_activity_data(self, activity_file_path, activity_args_file_path): """Load activity data from file""" activity_timeline_list = [] try: args_dict = {} with open(activity_args_file_path, 'r') as args_file: csv_reader = csv.reader(args_file) keys_list = next(csv_reader) # keys_list [name, type, op_full_name, stream_id, block_dim, grid_dim, ...] self._activity_keys_list = keys_list[1:3] + keys_list[4:6] for info in csv_reader: args_dict[info[0]] = info[1:3] + info[4:6] with open(activity_file_path, 'r') as f_obj: for line in f_obj: line_list = line.strip('\n').split(';') # concat activity args info. line_list += args_dict[line_list[0]] activity_timeline_list.append(line_list) except (IOError, OSError) as err: logger.critical('Error occurred when load activity timeline data intermediate file: %s', err) raise ProfilerIOException() return activity_timeline_list def init_timeline(self): """Init timeline metadata, adding all collected info.""" timeline_list = self._load_timeline_data() # Init a dict for counting the num of streams. stream_count_dict = {} for timeline in timeline_list: self._parse_timeline_data(timeline, 0) # Updating the collection of streams. if len(timeline) == 4: self._update_num_of_streams(timeline, stream_count_dict) # Add format thread meta data. self._format_meta_data_list.extend(self._timeline_meta) self._timeline_meta = self._format_meta_data_list # Update timeline summary info self._timeline_summary['num_of_streams'] += len(stream_count_dict.keys()) def check_op_name(self, op_name): """ Check whether the operator name exists. Args: op_name (str): The operator name or operator name prefix. Returns: bool, `True` if the operator name does exist, else `False`. """ if not op_name: raise ProfilerParamValueErrorException('The op_name should exist.') for op_time_info in self._timeline_meta: full_op_name = op_time_info['name'] if full_op_name and full_op_name.startswith(op_name): return True return False def _get_step_time_list_from_step_trace(self): """Produce the time of each step based on step_trace_profiling file.""" # Record the time of each step. step_time_list = [] step_start_op_name = [] step_end_op_name = [] step_num = 1 tid = "Steps" step_trace_profiling_path = self._get_and_validate_path( self._step_trace_original_filename ) try: with open(step_trace_profiling_path, 'r') as f_obj: for line in f_obj: line = line.strip().split() step_start_op_name.append(line[0].split(',')[0]) step_end_op_name.append(line[3].split(',')[0]) cur_step_start_time = float(line[0].split(',')[1]) cur_step_end_time = float(line[3].split(',')[1]) # convert duration time unit from ns to us. cur_step_duration_time = (cur_step_end_time - cur_step_start_time) / 1e3 step_time_item = [str(step_num), tid, cur_step_start_time, cur_step_duration_time] step_time_list.append(step_time_item) step_num += 1 except (IOError, OSError) as err: logger.critical(f'Error occurred when read {step_trace_profiling_path}: {err}') raise ProfilerIOException() return step_time_list def is_gpu_kernel_async_launch(self): """Recognize the solution that launch the gpu kernel async.""" step_trace_profiling_path = self._get_and_validate_path( self._step_trace_original_filename ) try: with open(step_trace_profiling_path, 'r') as f_obj: line = next(f_obj) first_string = line.strip().split()[0] # the data format of launch the gpu kernel async is "Default/op1,160123 op-name" # otherwise, the data format is "Default/op1 160123,12 " return bool(len(first_string.split(',')) == 2) except (IOError, OSError) as err: logger.critical(f'Error occurred when read {step_trace_profiling_path}: {err}') raise ProfilerIOException() class AscendTimelineGenerator(BaseTimelineGenerator): """Generate ascend Timeline data from file.""" _display_filename = 'ascend_timeline_display_{}.json' _timeline_summary_filename = 'ascend_timeline_summary_{}.json' _cluster_analyse_filename = 'ascend_cluster_analyse_{}_{}_{}_{}.csv' def __init__(self, profiling_dir, device_id, rank_id, rank_size): self._profiling_dir = profiling_dir self._device_id = device_id self._rank_id = rank_id self._tid_dict = { "computation_op": (self._MERGED_COMPUTATION_TID, self._OP_OVERLAP_PID), "communication_not_overlapped": (self._PURE_COMMUNICATION_TID, self._OP_OVERLAP_PID), "communication": (self._MERGED_COMMUNICATION_TID, self._OP_OVERLAP_PID), "free_time": (self._FREE_TIME_TID, self._OP_OVERLAP_PID) } self._rank_size = rank_size self._display_filename = self._display_filename.format(rank_id) self._timeline_summary_filename = self._timeline_summary_filename.format(rank_id) def _load_timeline_data(self): """Load timeline data from file.""" file_path = os.path.join( self._profiling_dir, self._output_timeline_data_file_path.format(self._rank_id) ) file_path = validate_and_normalize_path(file_path) if not os.path.exists(file_path): logger.critical("Failed to find parsed timeline file.") raise ProfilerFileNotFoundException('parsed timeline file') timeline_list = [] try: with open(file_path, 'r') as f_obj: for line in f_obj: if not line.startswith('op_name'): line_list = line.strip('\n').split(',') line_list[self._tid_idx] = f"Stream #{line_list[self._tid_idx]}" timeline_list.append(line_list) except (IOError, OSError) as err: logger.critical('Error occurred when read timeline intermediate file: %s', err) raise ProfilerIOException() return timeline_list def _parse_timeline_data(self, timeline, min_cycle_counter): """Parse timeline data.""" # factor to convert the time unit from 1ms to 1us for timeline display factor = 1000 op_meta = TimelineContainer(timeline) timeline_dict = {} timeline_dict['name'] = op_meta.op_name.split('/')[-1] timeline_dict['ph'] = 'X' timeline_dict['tid'] = op_meta.stream_id timeline_dict['ts'] = (op_meta.start_time - min_cycle_counter) * factor dur = op_meta.duration * factor timeline_dict['dur'] = dur if op_meta.pid is None: timeline_dict['pid'] = int(self._device_id) # Update total time of operator execution. if op_meta.stream_id not in ["Steps", "Scope Name"]: self._timeline_summary['total_time'] += op_meta.duration else: # AllReduce and AI CPU pid timeline_dict['pid'] = op_meta.pid if op_meta.stream_id == "Scope Name": # remove the level of scope name which has a format like "0-conv2-Conv2d". timeline_dict['name'] = "-".join(op_meta.op_name.split('-')[1:]) timeline_dict['scope_level'] = int(op_meta.op_name.split('-')[0]) elif op_meta.stream_id == self._host_cpu_op_label: timeline_dict['pid'] = self._HOST_CPU_PID self._update_format_meta_data(timeline_dict) self._timeline_meta.append(timeline_dict) def init_timeline(self, communication_info, framework_info, aicpu_info, min_cycle_counter, source_path): """ Init timeline metadata, adding all collected info. Args: communication_info (list[list]): The metadata of communication operator. framework_info (dict): The framework metadata. aicpu_info (dict): The metadata of AI CPU operator. min_cycle_counter (float): The minimum cycle counter of the timeline. source_path (str): The source of file. """ if min_cycle_counter == float('inf'): min_cycle_counter = 0 logger.info('Initiating timeline...') timeline_list = self._load_timeline_data() cpu_timeline_generator = CpuTimelineGenerator(self._profiling_dir, self._rank_id) cpu_timeline_list = cpu_timeline_generator.get_timeline_data() if cpu_timeline_list: self._clock_synchronize_to_device(cpu_timeline_list, source_path) timeline_list.extend(cpu_timeline_list) timeline_list.sort(key=lambda x: float(x[self._start_time_idx])) self._max_scope_name_num = self._get_max_scope_name_num(timeline_list) self._timeline_summary['op_exe_times'] = len(timeline_list) self._timeline_summary['max_scope_name_num'] = self._max_scope_name_num # Generate step time. self._set_step_start_and_end_op_name(timeline_list) step_time_list = self._get_step_time_list(timeline_list) # Add Scope Name. default_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Default") gradient_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Gradients") recompute_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "recompute_Default") # Add AI CPU data into timeline temp list and sort by start time. aicpu_data = aicpu_info.get('info') if aicpu_data: timeline_list.extend(aicpu_data) self._timeline_summary['op_exe_times'] += aicpu_info.get('op_exe_times', 0) self._timeline_summary['num_of_streams'] += aicpu_info.get('num_of_streams', 0) self._timeline_summary['num_of_ops'] += aicpu_info.get('num_of_ops', 0) self._timeline_summary['total_time'] += aicpu_info.get('total_time', 0) timeline_list.sort(key=lambda x: float(x[self._start_time_idx])) # Add AllReduce info to timeline temp list and sort by start time. if communication_info: logger.debug('AllReduce info found. Start adding info into timeline...') cluster_related_timeline = self._analyse_and_write_cluster_profiling_data( timeline_list, communication_info, step_time_list) timeline_list.extend(cluster_related_timeline) timeline_list.extend(communication_info) timeline_list.sort(key=lambda x: float(x[self._start_time_idx])) # Add step time and scope name info. timeline_list.extend(step_time_list) timeline_list.extend(default_scope_name_time_list) timeline_list.extend(recompute_scope_name_time_list) timeline_list.extend(gradient_scope_name_time_list) timeline_list.sort(key=lambda x: float(x[self._start_time_idx])) # Init a dict for counting the num of streams. stream_count_dict = {} for timeline in timeline_list: self._parse_timeline_data(timeline, min_cycle_counter) # Updating the collection of streams. if len(timeline) == 4: self._update_num_of_streams(timeline, stream_count_dict) # Add format thread meta data. self._format_meta_data_list.extend(self._timeline_meta) self._timeline_meta = self._format_meta_data_list # Get framework metadata. framework_obj_list = framework_info.get('object') # The length of list is the number of operators. self._timeline_summary['num_of_ops'] += len(framework_obj_list) self._add_framework_info(framework_obj_list) logger.info('Finished adding info into timeline...') # Update timeline summary info self._timeline_summary['num_of_streams'] += len(stream_count_dict.keys()) def _clock_synchronize_to_device(self, timeline_list, source_path): """Synchronize the timestamp from host to device.""" host_start_file_path = os.path.join(source_path, f"host_start.log.{self._device_id}") dev_start_file_path = os.path.join(source_path, f"dev_start.log.{self._device_id}") try: with open(host_start_file_path) as f_obj: lines = f_obj.readlines() # lines[2] stores host monotonic_raw time of start training. host_monotonic = int(lines[2].strip().split(':')[1]) except (IOError, OSError) as err: logger.critical('Error occurred when read host_start.log: %s', err) raise ProfilerIOException() try: with open(dev_start_file_path) as f_obj: lines = f_obj.readlines() # lines[2] stores device cycle counter of start training. dev_cntvct = int(lines[2].strip().split(':')[1]) except (IOError, OSError) as err: logger.critical('Error occurred when read dev_start.log: %s', err) raise ProfilerIOException() factor_ns_to_ms = 1e-6 factor_ten_ns_to_ns = 10 factor_ms_to_ns = 1e6 for idx, time_item in enumerate(timeline_list): host_time = int(float(time_item[self._start_time_idx]) * factor_ms_to_ns) device_time = dev_cntvct * factor_ten_ns_to_ns + (host_time - host_monotonic) timeline_list[idx][self._start_time_idx] = device_time * factor_ns_to_ms def _add_framework_info(self, framework_obj_list): """ Add framework info into timeline metadata. Args: framework_obj_list (list): The framework metadata. """ logger.debug('Start adding framework info into timeline...') # Get the framework info that will be written into timeline. framework_info_dict = {} for framework_obj in framework_obj_list: op_name = framework_obj[0] op_type = framework_obj[1] op_full_name = framework_obj[4] op_info = framework_obj[5] framework_info_dict[op_full_name] = { 'name': op_name, 'args': { 'type': op_type, 'fullname': op_full_name } } framework_info_dict[op_full_name]['args'].update(op_info) # Insert framework info into timeline. for timeline_item in self._timeline_meta: op_full_name = timeline_item.get('name') framework_item = framework_info_dict.get(op_full_name) if framework_item: timeline_item['name'] = framework_item.get('name') timeline_item['args'] = framework_item.get('args') logger.debug('Finished adding framework info into timeline...') def _produce_two_separated_timeline(self, timeline, op_name): """Produce two separated timeline based on op_name.""" timeline_include_op_name = [] timeline_exclude_op_name = [] for time_item in timeline: if op_name in time_item[self._op_name_idx]: timeline_include_op_name.append(time_item) else: timeline_exclude_op_name.append(time_item) return timeline_include_op_name, timeline_exclude_op_name def _analyse_and_write_cluster_profiling_data(self, aicore_timeline, communication_timeline, step_time_list): """ Analyse the cluster communication and computation data, and write result to file. To analyse the cluster performance bottleneck based on timeline, define the time of a training step as "t_total", propose five metrics as follows: 1) The time that "receive" operators not overlapped by others(t1) 2) The time that is consumed inside the stage(t_total - t1) 3) The time that "communication" operators not overlapped by others(t2) 4) The time that consumed by computation(t_total - t2) 5) The time that "collective communication" operators not overlapped by others(t3) In pipeline parallel mode, we can locate slow stage based on t_total - t1. Inside each stage, we can locate slow card based on t_total - t2. The value of t1 indicates the degree that communication time between stages slow down the training. The value of t3 indicates the degree that communication inside each stage slow down the training. """ step_num = len(step_time_list) is_pipeline_parallel = False comm_merged_timeline, _, comm_display_timeline = self._get_merged_time_list( communication_timeline, display_name="communication" ) aicore_timeline_interval, _, aicore_display_timeline = self._get_merged_time_list( aicore_timeline, get_interval_time=True ) # Consider if the overlap will be 0 or not. comm_not_overlaped_timeline = self._get_intersection_time( aicore_timeline_interval, comm_merged_timeline ) # Process receive part. all_timeline = aicore_timeline + communication_timeline all_timeline.sort(key=lambda x: float(x[self._start_time_idx])) receive_op_timeline, timeline_exclude_receive_op = self._produce_two_separated_timeline( all_timeline, "Receive-op" ) if receive_op_timeline: is_pipeline_parallel = True receive_op_merged_timeline = self._get_merged_time_list(receive_op_timeline)[0] timeline_exclude_receive_op_interval = self._get_merged_time_list( timeline_exclude_receive_op, get_interval_time=True )[0] receive_op_not_overlaped_timeline = self._get_intersection_time( timeline_exclude_receive_op_interval, receive_op_merged_timeline ) # Process collective communication part. collective_comm_timeline = self._produce_two_separated_timeline( communication_timeline, "Receive-op" )[-1] collective_comm_merged_timeline = self._get_merged_time_list(collective_comm_timeline)[0] collective_comm_not_overlaped_timeline = self._get_intersection_time( aicore_timeline_interval, collective_comm_merged_timeline ) # Generate free time that exclude computation and communication time. free_timeline = self._get_merged_time_list( all_timeline, get_interval_time=True, display_name="free_time" )[1] try: # Compute these five metrics mentioned above per step. recieve_alone_time = self._compute_time_inside_step(receive_op_not_overlaped_timeline, step_time_list) stage_time, computation_time = [], [] comm_alone_time = self._compute_time_inside_step(comm_not_overlaped_timeline, step_time_list) collective_comm_alone_time = self._compute_time_inside_step( collective_comm_not_overlaped_timeline, step_time_list ) for step in range(step_num): if is_pipeline_parallel: stage_time.append(step_time_list[step][self._duration_idx] - recieve_alone_time[step]) computation_time.append(step_time_list[step][self._duration_idx] - comm_alone_time[step]) metrices_per_step_list = [computation_time, comm_alone_time, stage_time, recieve_alone_time, collective_comm_alone_time] if step_num > 1: for metric in metrices_per_step_list: metric.append(sum(metric[1:]) / (step_num - 1)) self._write_cluster_metrices(metrices_per_step_list, is_pipeline_parallel) except IndexError as e: logger.error(e) res_timeline = [] res_timeline.extend(comm_not_overlaped_timeline) res_timeline.extend(aicore_display_timeline) res_timeline.extend(comm_display_timeline) res_timeline.extend(free_timeline) return res_timeline def _write_cluster_metrices(self, metrices, is_pipeline_parallel): """Write cluster metric.""" # Note that the feature of cluster bottleneck analyse is not supported in offline parse mode, # due to that parallel context is not set. try: parallel_mode = get_auto_parallel_context("parallel_mode") stage_num = get_auto_parallel_context("pipeline_stages") except RuntimeError: logger.warning("[profiler] the feature of cluster bottleneck analyse " "is not supported in offline parse mode.") parallel_mode = "data_parallel" stage_num = 1 if stage_num > 1: parallel_mode = "pipeline-parallel" elif parallel_mode != "data_parallel": parallel_mode = "model-parallel" else: parallel_mode = "data-parallel" cluster_analyse_file_path = os.path.join( self._profiling_dir, self._cluster_analyse_filename.format(parallel_mode, stage_num, self._rank_size, self._rank_id) ) cluster_analyse_file_path = validate_and_normalize_path(cluster_analyse_file_path) try: with open(cluster_analyse_file_path, 'w') as file_handle: csv_writer = csv.writer(file_handle) if is_pipeline_parallel: header = ['computation_time', 'communication_alone_time', 'stage_time', 'receive_alone_time', 'collective_communication_alone_time'] zip_metrices = zip(metrices[0], metrices[1], metrices[2], metrices[3], metrices[4]) else: header = ['computation_time', 'communication_alone_time'] zip_metrices = zip(metrices[0], metrices[1]) csv_writer.writerow(header) for row_data in zip_metrices: row_data = [round(val, 4) for val in row_data] csv_writer.writerow(row_data) os.chmod(cluster_analyse_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.warning(f'Failed to save {cluster_analyse_file_path}. {err}') raise ProfilerIOException def _compute_time_inside_step(self, metric_timeline, step_time_list): """Compute per step time of metric_timeline.""" per_step_time_list = [] step = 0 cur_step_metric_time = 0 step_end_time = step_time_list[step][self._start_time_idx] + \ step_time_list[step][self._duration_idx] for time_item in metric_timeline: start_time = time_item[self._start_time_idx] if start_time > step_end_time: per_step_time_list.append(cur_step_metric_time) step += 1 step_end_time = step_time_list[step][self._start_time_idx] + \ step_time_list[step][self._duration_idx] cur_step_metric_time = 0 cur_step_metric_time += time_item[self._duration_idx] per_step_time_list.append(cur_step_metric_time) return per_step_time_list def _get_merged_time_list(self, time_list, get_interval_time=False, display_name="computation_op"): """ Get merged time segment list. The process of merge is, for example, there is a list [[1,5], [2,6], [7,8]], each items in this list contains a start_time and end_time, the merged result is [[1,6], [7,8]]. """ time_merged_segment_list = [] tid = self._tid_dict[display_name][0] pid = self._tid_dict[display_name][1] for time_item in time_list: time_segment = list(map(float, time_item[self._start_time_idx:self._duration_idx + 1])) time_segment[1] += time_segment[0] if not time_merged_segment_list or \ time_segment[0] > time_merged_segment_list[-1]: time_merged_segment_list.extend(time_segment) else: time_merged_segment_list[-1] = max( time_merged_segment_list[-1], time_segment[1] ) # merged_display_list data used for ui page. merged_display_list = [ [display_name, tid, time_merged_segment_list[i * 2], time_merged_segment_list[i * 2 + 1] - time_merged_segment_list[i * 2], pid] for i in range(len(time_merged_segment_list) // 2) ] if get_interval_time: time_merged_segment_list = time_merged_segment_list[1:-1] # merged_res_list data used to compute overlap with other time_list. merged_res_list = [ [display_name, tid, time_merged_segment_list[i * 2], time_merged_segment_list[i * 2 + 1], pid] for i in range(len(time_merged_segment_list) // 2) ] # interval_display_list is interval time used for ui page. interval_display_list = [ [display_name, tid, time_merged_segment_list[i * 2], time_merged_segment_list[i * 2 + 1] - time_merged_segment_list[i * 2], pid] for i in range(len(time_merged_segment_list) // 2) ] return merged_res_list, interval_display_list, merged_display_list def _get_intersection_time(self, first_time_list, second_time_list, display_name="communication_not_overlapped"): """Get intersection time of two time list.""" first_list_idx, second_list_idx = 0, 0 first_list_len = len(first_time_list) second_list_len = len(second_time_list) intersection_segment_display_list = [] while first_list_idx < first_list_len and \ second_list_idx < second_list_len: intersection_start = max( first_time_list[first_list_idx][self._start_time_idx], second_time_list[second_list_idx][self._start_time_idx] ) intersection_end = min( first_time_list[first_list_idx][self._duration_idx], second_time_list[second_list_idx][self._duration_idx] ) if intersection_start < intersection_end: intersection_segment_display_list.append( [display_name, self._tid_dict[display_name][0], intersection_start, intersection_end - intersection_start, self._tid_dict[display_name][1]] ) if first_time_list[first_list_idx][self._duration_idx] >= \ second_time_list[second_list_idx][self._duration_idx]: second_list_idx += 1 else: first_list_idx += 1 return intersection_segment_display_list class CpuTimelineGenerator(GpuTimelineGenerator): """Generate cpu Timeline data from file.""" _output_op_execute_time_file_path = "cpu_op_execute_timestamp_{}.txt" def _get_and_validate_path(self, file_name): """Generate op or activity file path from file name, and validate this path.""" file_path = os.path.join( self._profiling_dir, file_name.format(self._device_id) ) file_path = validate_and_normalize_path(file_path) return file_path def load_cpu_op_data(self): """Load cpu operator data from file""" op_file_path = self._get_and_validate_path( self._output_op_execute_time_file_path) timeline_list = [] if not os.path.exists(op_file_path): logger.info("No cpu operator info.") return timeline_list timeline_list = self._load_op_data(op_file_path) factor_ms_to_us = 1e-3 for time_item in timeline_list: time_item[self._duration_idx] = float(time_item[self._duration_idx]) / factor_ms_to_us return timeline_list def get_timeline_data(self): """Get timeline data from file.""" timeline_list = self.load_cpu_op_data() factor_ns_to_ms = 1e6 factor_us_to_ms = 1e3 for time_item in timeline_list: time_item[self._start_time_idx] = float(time_item[self._start_time_idx]) / factor_ns_to_ms time_item[self._duration_idx] = float(time_item[self._duration_idx]) / factor_us_to_ms return timeline_list