diff --git a/tests/st/func/profiler/test_minddata_pipeline_analyser.py b/tests/st/func/profiler/test_minddata_pipeline_analyser.py new file mode 100644 index 00000000..4c25daa6 --- /dev/null +++ b/tests/st/func/profiler/test_minddata_pipeline_analyser.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Fuction: + Test profiler to watch the performance of training. +Usage: + pytest tests/st/func/profiler +""" +import os +import shutil +from unittest import mock + +import pytest + +from mindinsight.profiler import Profiler +from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory +from mindinsight.profiler.parser.framework_parser import FrameworkParser +from tests.st.func.profiler.conftest import BASE_SUMMARY_DIR +from tests.ut.profiler import RAW_DATA_BASE + + +@pytest.mark.usefixtures('create_summary_dir') +class TestMinddataPipelineAnalyser: + """Test minddata pipeline analyser module.""" + JOB_ID = 'JOB3' + + @classmethod + def setup_class(cls): + """Generate parsed files.""" + cls.generate_parsed_files() + + def setup_method(self): + """Create analyser.""" + self._analyser = AnalyserFactory.instance().get_analyser( + 'minddata_pipeline', self.profiler, '1') + + @classmethod + def generate_parsed_files(cls): + """Test parse raw info about profiler.""" + cls.summary_dir = os.path.join(BASE_SUMMARY_DIR, 'normal_run') + cls.profiler = os.path.join(cls.summary_dir, 'profiler') + FrameworkParser._raw_data_dir = RAW_DATA_BASE + if not os.path.exists(cls.summary_dir): + os.makedirs(cls.summary_dir) + os.makedirs(cls.profiler, exist_ok=True) + pipeline_path = os.path.join(RAW_DATA_BASE, 'profiler', 'pipeline_profiling_1.json') + shutil.copy(pipeline_path, cls.profiler) + Profiler._base_profiling_container_path = os.path.join(RAW_DATA_BASE, 'container') + with mock.patch('mindinsight.profiler.profiling.PROFILING_LOG_BASE_PATH', RAW_DATA_BASE): + profiler = Profiler(subgraph='all', is_detail=True, is_show_op_path=False, + output_path=cls.summary_dir, job_id=cls.JOB_ID) + profiler.analyse() + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + def test_query(self): + """Test the function of querying minddata pipeline infomation.""" + expect_result = { + 'col_name': [ + 'op_id', 'op_type', 'output_queue_average_size', 'output_queue_length', + 'output_queue_usage_rate', 'sample_interval', 'parent_id', 'children_id' + ], + 'object': [ + [1, 'Shuffle', 20.0, 64, 0.3125, 10, 0, [2, 3]], + [2, 'TFReader', 20.0, 64, 0.3125, 10, 1, None], + [3, 'TFReader', 20.0, 64, 0.3125, 10, 1, None], + [0, 'Batch', None, None, None, 10, None, [1]] + ], + 'size': 4 + } + condition = { + 'sort_condition': { + 'name': 'output_queue_average_size', + 'type': 'descending' + } + } + result = self._analyser.query(condition) + assert expect_result == result + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + def test_get_op_and_parent_op_info(self): + """Test the function of the target operator and queue infomation.""" + expect_result = { + 'current_op': { + 'op_id': 1, + 'op_type': 'Shuffle', + 'num_workers': 1 + }, + 'parent_op': { + 'op_id': 0, + 'op_type': 'Batch', + 'num_workers': 4 + }, + 'queue_info': { + 'output_queue_size': [10, 20, 30], + 'output_queue_average_size': 20.0, + 'output_queue_length': 64, + 'output_queue_usage_rate': 0.3125, + 'sample_interval': 10 + } + } + result = self._analyser.get_op_and_parent_op_info(1) + assert expect_result == result diff --git a/tests/st/func/profiler/test_op_analyser.py b/tests/st/func/profiler/test_op_analyser.py new file mode 100644 index 00000000..7e32a58a --- /dev/null +++ b/tests/st/func/profiler/test_op_analyser.py @@ -0,0 +1,225 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Fuction: + Test profiler to watch the performance of training. +Usage: + pytest tests/st/func/profiler +""" +import os +from unittest import mock + +import pytest + +from mindinsight.profiler import Profiler +from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory +from mindinsight.profiler.parser.framework_parser import FrameworkParser +from tests.st.func.profiler.conftest import BASE_SUMMARY_DIR +from tests.ut.profiler import RAW_DATA_BASE + + +OP_GATHER_V2_INFO = { + 'col_name': [ + 'op_name', 'op_type', 'execution_time', 'subgraph', 'full_op_name', 'op_info' + ], + 'object': [ + [ + 'GatherV2-op55', 'GatherV2', 42.220212142857136, 'Default', + 'Default/network-TrainStepWrap/network-VirtualDatasetCellTriple/' + '_backbone-NetWithLossClass/network-WideDeepModel/GatherV2-op55', + { + 'input_0': { + 'format': 'DefaultFormat', + 'data_type': 'NUMBER_TYPE_FLOAT32', + 'shape': '184696,8' + }, + 'input_1': { + 'format': 'DefaultFormat', + 'data_type': 'NUMBER_TYPE_INT32', + 'shape': '128000,39' + }, + 'output_0': { + 'format': 'DefaultFormat', + 'data_type': 'NUMBER_TYPE_FLOAT32', + 'shape': '128000,39,8' + } + } + ], + [ + 'GatherV2-op33', 'GatherV2', 0.9352293333333332, 'Default', + 'Default/network-TrainStepWrap/network-VirtualDatasetCellTriple/' + '_backbone-NetWithLossClass/network-WideDeepModel/GatherV2-op33', + { + 'input_0': { + 'format': 'DefaultFormat', + 'data_type': 'NUMBER_TYPE_FLOAT32', + 'shape': '184696,1' + }, + 'input_1': { + 'format': 'DefaultFormat', + 'data_type': 'NUMBER_TYPE_INT32', + 'shape': '16000,39' + }, + 'output_0': { + 'format': 'DefaultFormat', + 'data_type': 'NUMBER_TYPE_FLOAT32', + 'shape': '16000,39,1' + } + } + ] + ], + 'size': 2 +} + + +@pytest.mark.usefixtures('create_summary_dir') +class TestOpAnalyser: + """Test AICORE and AICPU analyser module.""" + JOB_ID = 'JOB3' + + @classmethod + def setup_class(cls): + """Generate parsed files.""" + cls.generate_parsed_files() + + def setup_method(self): + """Create analyser.""" + self._analyser_aicore_type = AnalyserFactory.instance().get_analyser( + 'aicore_type', self.profiler, '1') + self._analyser_aicore_detail = AnalyserFactory.instance().get_analyser( + 'aicore_detail', self.profiler, '1') + + @classmethod + def generate_parsed_files(cls): + """Test parse raw info about profiler.""" + cls.summary_dir = os.path.join(BASE_SUMMARY_DIR, 'normal_run') + cls.profiler = os.path.join(cls.summary_dir, 'profiler') + FrameworkParser._raw_data_dir = RAW_DATA_BASE + if not os.path.exists(cls.summary_dir): + os.makedirs(cls.summary_dir) + Profiler._base_profiling_container_path = os.path.join(RAW_DATA_BASE, 'container') + with mock.patch('mindinsight.profiler.profiling.PROFILING_LOG_BASE_PATH', RAW_DATA_BASE): + profiler = Profiler(subgraph='all', is_detail=True, is_show_op_path=False, + output_path=cls.summary_dir, job_id=cls.JOB_ID) + profiler.analyse() + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + def test_query_aicore_type_1(self): + """Test the function of querying AICORE operator type infomation.""" + expect_result = { + 'col_name': ['op_type', 'execution_time', 'execution_frequency', 'percent'], + 'object': [ + ['UnsortedSegmentSum', 44.60782642857142, 2, 35.28], + ['GatherV2', 43.15544147619047, 2, 34.13], + ['Slice', 20.376314999999998, 16, 16.12], + ['Concat', 5.80845380952381, 4, 4.59], + ['Split', 2.7142774761904764, 2, 2.15], + ['MatMul', 1.9366814285714287, 15, 1.53], + ['Mul', 1.9029486666666666, 32, 1.51], + ['StridedSliceGrad', 1.5068342857142858, 2, 1.19], + ['TransData', 1.1151575238095237, 30, 0.88], + ['ReluGrad', 0.8540685714285714, 5, 0.68], + ['Cast', 0.4846848571428572, 15, 0.38], + ['ReLU', 0.48328214285714277, 5, 0.38], + ['RealDiv', 0.4228071904761905, 15, 0.33], + ['StridedSlice', 0.3455687619047618, 2, 0.27], + ['Adam', 0.2859357142857143, 11, 0.23], + ['BiasAdd', 0.18966285714285713, 5, 0.15], + ['BiasAddGrad', 0.07168142857142856, 5, 0.06], + ['Tile', 0.04415833333333334, 4, 0.03], + ['ReduceSum', 0.030764857142857142, 5, 0.02], + ['ApplyFtrl', 0.025453571428571426, 2, 0.02], + ['AtomicAddrClean', 0.019368666666666666, 8, 0.02], + ['AddN', 0.012836428571428572, 1, 0.01], + ['Square', 0.009799333333333334, 1, 0.01], + ['SigmoidCrossEntropyWithLogitsGrad', 0.009582142857142859, 2, 0.01], + ['TensorAdd', 0.009218380952380952, 3, 0.01], + ['SigmoidCrossEntropyWithLogits', 0.004808571428571428, 1, 0.0], + ['ReduceMean', 0.004534999999999999, 1, 0.0], + ['Assign', 0.0024766666666666665, 2, 0.0], + ['AssignAdd', 0.001688, 1, 0.0] + ], + 'size': 29 + } + condition = { + 'sort_condition': { + 'name': 'execution_time', + 'type': 'descending' + } + } + result = self._analyser_aicore_type.query(condition) + assert expect_result == result + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + def test_query_aicore_type_2(self): + """Test the function of querying AICORE operator type infomation.""" + expect_result = { + 'col_name': ['op_type', 'execution_time', 'execution_frequency', 'percent'], + 'object': [ + ['MatMul', 1.9366814285714287, 15, 1.53], + ['Mul', 1.9029486666666666, 32, 1.51] + ], + 'size': 2 + } + condition = { + 'filter_condition': { + 'op_type': { + 'partial_match_str_in': ['Mul'] + } + }, + 'sort_condition': { + 'name': 'execution_time', + 'type': 'descending' + } + } + result = self._analyser_aicore_type.query(condition) + assert expect_result == result + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + def test_query_aicore_detail_1(self): + """Test the function of querying AICORE operator detail infomation.""" + expect_result = OP_GATHER_V2_INFO + condition = { + 'filter_condition': { + 'op_type': { + 'in': ['GatherV2'] + } + }, + 'sort_condition': { + 'name': 'execution_time', + 'type': 'descending' + }, + 'group_condition': { + 'limit': 10, + 'offset': 0 + } + } + result = self._analyser_aicore_detail.query(condition) + assert expect_result == result diff --git a/tests/utils/resource/profiler/pipeline_profiling_1.json b/tests/utils/resource/profiler/pipeline_profiling_1.json new file mode 100644 index 00000000..2f5fdca7 --- /dev/null +++ b/tests/utils/resource/profiler/pipeline_profiling_1.json @@ -0,0 +1,55 @@ +{ + "sampling_interval": 10, + "op_info": [ + { + "op_id": 4, + "op_type": "TFReader", + "num_workers": 4, + "metrics": null, + "children": [3] + }, + { + "op_id": 3, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 2, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 1, + "op_type": "Shuffle", + "num_workers": 1, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": [2, 4] + }, + { + "op_id": 0, + "op_type": "Batch", + "num_workers": 4, + "metrics": null, + "children": [1] + } + ] +} \ No newline at end of file