fix security and non-security issues

fix security and non-security issues of Profile Python code

mindspore/profiler/parser/aicpu_data_parser.py

@@ -16,6 +16,7 @@
 The parser for AI CPU preprocess data.
 """
 import os
+import stat
 
 from mindspore.profiler.common.util import fwrite_format, get_file_join_name
 from mindspore import log as logger
@@ -96,7 +97,7 @@ class DataPreProcessParser:
             ai_cpu_str = str(ai_cpu_data.read().replace(b'\n\x00', b' ___ ')
                              .replace(b'\x00', b' ___ '))[2:-1]
             ai_cpu_lines = ai_cpu_str.split(" ___ ")
-
+        os.chmod(self._source_file_name, stat.S_IREAD | stat.S_IWRITE)
         result_list = list()
         ai_cpu_total_time_summary = 0
         # Node serial number.

mindspore/profiler/parser/framework_parser.py

@@ -18,6 +18,7 @@ import enum
 import json
 import os
 import re
+import stat
 
 from mindspore.profiler.common.exceptions.exceptions import \
     ProfilerPathErrorException, ProfilerDirNotFoundException, \
@@ -447,6 +448,7 @@ class FrameworkParser:
     def _parse_task_files(self):
         """Parse the framework task files."""
         for path in self._framework_path['task']:
+            path = validate_and_normalize_path(path)
             with open(path, 'r') as file:
                 for task_info in file:
                     infos = task_info.strip('\n').split(' ')
@@ -490,6 +492,7 @@ class FrameworkParser:
                 value.append(key)
                 value.extend(none_list)
                 csv_writer.writerow(value)
+        os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
 
     def _parse_one_row_graph_info(self, row_info):
         """
@@ -591,6 +594,7 @@ class FrameworkParser:
     def _parse_point_files(self):
         """Parse the framework point files."""
         for path in self._framework_path['point']:
+            path = validate_and_normalize_path(path)
             with open(path, 'r') as file:
                 for point_info in file:
                     infos = point_info.strip('\n').split(' ')

mindspore/profiler/parser/hwts_log_parser.py

@@ -17,7 +17,8 @@ import os
 import struct
 from mindspore.profiler.common.util import fwrite_format, get_file_join_name
 from mindspore import log as logger
-
+from mindspore.profiler.common.validator.validate_path import \
+    validate_and_normalize_path
 
 class HWTSLogParser:
     """
@@ -68,8 +69,10 @@ class HWTSLogParser:
 
         result_data = ""
 
+        self._source_flie_name = validate_and_normalize_path(self._source_flie_name)
         with open(self._source_flie_name, 'rb') as hwts_data:
             while True:
+                # read 64 bit data
                 line = hwts_data.read(64)
                 if line:
                     if not line.strip():
@@ -77,6 +80,8 @@ class HWTSLogParser:
                 else:
                     break
                 byte_first_four = struct.unpack('BBHHH', line[0:8])
+                # byte_first[0:4] refers to count. byte_first[4] refers to is_warn_res0_0v.
+                # byte_first[5:8] refers to the type of ms.
                 byte_first = bin(byte_first_four[0]).replace('0b', '').zfill(8)
                 ms_type = byte_first[-3:]
                 is_warn_res0_ov = byte_first[4]

mindspore/profiler/parser/integrator.py

@@ -16,6 +16,7 @@
 import csv
 import json
 import os
+import stat
 from decimal import Decimal
 
 from mindspore import log as logger
@@ -91,6 +92,7 @@ class Integrator:
             self._profiling_dir,
             self._file_name_framework.format(self._device_id)
         )
+        framework_file = validate_and_normalize_path(framework_file)
         if not os.path.isfile(framework_file):
             return
 
@@ -130,6 +132,7 @@ class Integrator:
             self._profiling_dir,
             self._file_name_aicore_detail_time.format(self._device_id)
         )
+        aicore_detail_file = validate_and_normalize_path(aicore_detail_file)
         if not os.path.isfile(aicore_detail_file):
             return
 
@@ -169,6 +172,7 @@ class Integrator:
             self._profiling_dir,
             self._file_name_aicpu_time.format(self._device_id)
         )
+        aicpu_file = validate_and_normalize_path(aicpu_file)
         if not os.path.isfile(aicpu_file):
             return
 
@@ -197,6 +201,7 @@ class Integrator:
             self._profiling_dir,
             self._file_name_aicore_type_time.format(self._device_id)
         )
+        op_type_file_path = validate_and_normalize_path(op_type_file_path)
         if not os.path.isfile(op_type_file_path):
             logger.warning('The file <%s> does not exist.', op_type_file_path)
             return
@@ -217,6 +222,8 @@ class Integrator:
             self._profiling_dir,
             self._file_name_framework.format(self._device_id)
         )
+        op_detail_file_path = validate_and_normalize_path(op_detail_file_path)
+        framework_file_path = validate_and_normalize_path(framework_file_path)
         if not os.path.isfile(op_detail_file_path):
             logger.warning('The file <%s> does not exist.', op_detail_file_path)
             return
@@ -251,6 +258,7 @@ class Integrator:
         if not file_path:
             logger.error("Failed to find parsed trace time file.")
             raise ProfilerFileNotFoundException('parsed step trace time file')
+        file_path = validate_and_normalize_path(file_path)
         with open(file_path, 'r') as handle:
             csv_reader = csv.reader(handle)
             self.__column__ = next(csv_reader)
@@ -261,6 +269,7 @@ class Integrator:
     def _load_point_info(self):
         """Load point info."""
         file_path = os.path.join(self._profiling_dir, 'step_trace_point_info.json')
+        file_path = validate_and_normalize_path(file_path)
         if os.path.isfile(file_path):
             with open(file_path, 'r', encoding='utf-8') as file:
                 try:
@@ -540,6 +549,7 @@ class BaseTimelineGenerator:
                         break
                     json_file.write(',')
                 json_file.write(']')
+                os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.error('Error occurred when write timeline display file: %s', err)
             raise ProfilerIOException
@@ -556,6 +566,7 @@ class BaseTimelineGenerator:
         try:
             with open(timeline_summary_file_path, 'w') as json_file:
                 json.dump(self._timeline_summary, json_file)
+            os.chmod(timeline_summary_file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.error('Error occurred when write timeline summary file: %s', err)
             raise ProfilerIOException

mindspore/profiler/parser/minddata_parser.py

@@ -17,7 +17,8 @@ import os
 
 from mindspore.profiler.common.util import get_file_join_name, fwrite_format
 from mindspore import log as logger
-
+from mindspore.profiler.common.validator.validate_path import \
+    validate_and_normalize_path
 
 class MinddataParser:
     """Minddata Aicpu Parser."""
@@ -34,6 +35,7 @@ class MinddataParser:
         """
         result = list()
         try:
+            minddata_aicpu_source_path = validate_and_normalize_path(minddata_aicpu_source_path)
             with open(minddata_aicpu_source_path) as source_data_file:
                 source_data = source_data_file.read()
                 step_data = source_data.split("\x00")
@@ -77,6 +79,7 @@ class MinddataParser:
             device_id (str): the device id.
         """
         col_names = ["node_name", "start_time", "end_time", "queue_size"]
+        source_path = validate_and_normalize_path(source_path)
         minddata_aicpu_source_path = get_file_join_name(
             input_path=source_path, file_name='DATA_PREPROCESS.AICPUMI')
         if not minddata_aicpu_source_path:

mindspore/profiler/parser/minddata_pipeline_parser.py

@@ -16,6 +16,7 @@
 import csv
 import json
 import os
+import stat
 from queue import Queue
 
 from mindspore.profiler.common.exceptions.exceptions import \
@@ -167,6 +168,7 @@ class MinddataPipelineParser:
             self._parse_and_save_op_info(
                 csv_writer, op_id_info_cache, sample_interval
             )
+        os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
 
     def _parse_and_save_op_info(self, csv_writer, op_id_info_cache,
                                 sample_interval):

mindspore/profiler/parser/optime_parser.py

@@ -14,6 +14,7 @@
 # ============================================================================
 """Op compute time files parser."""
 import os
+import stat
 from mindspore.profiler.common.util import fwrite_format
 from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
     ProfilerIOException
@@ -163,6 +164,7 @@ class OPComputeTimeParser:
                 for timeline in timeline_data:
                     timeline = [str(item) for item in timeline]
                     f_obj.write(','.join(timeline) + '\n')
+            os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.error('Error occurred when writing intermediate timeline file: %s', err)
             raise ProfilerIOException

mindspore/profiler/parser/step_trace_parser.py

@@ -25,6 +25,8 @@ from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorExc
     JobIdMismatchException, ProfilerIOException
 from mindspore import log
 from mindspore.profiler.common.util import get_summary_for_step_trace
+from mindspore.profiler.common.validator.validate_path import \
+    validate_and_normalize_path
 
 StepTraceStruct = namedtuple(
     'TrainingTraceStruct', ['tag_id', 'task_id', 'stream_id', 'sys_count']
@@ -199,6 +201,7 @@ class StepTraceParser:
         log.info("Start to parse step trace file.")
         event_info = {}
         for source_file in source_files:
+            source_file = validate_and_normalize_path(source_file)
             with open(source_file, 'rb') as handler:
                 content = handler.read()
                 for step_trace in self._get_next_step_trace(content, event_info):

mindspore/profiler/profiling.py

@@ -152,11 +152,10 @@ class Profiler:
             # parse hwts.log.data.45.dev file, and get task profiling data
             hwts_output_filename = self._hwts_output_filename_target + self._dev_id + ".txt"
             hwts_output_filename = os.path.join(self._output_path, hwts_output_filename)
+            source_path = validate_and_normalize_path(source_path)
+            hwts_output_filename = validate_and_normalize_path(hwts_output_filename)
             hwtslog_parser = HWTSLogParser(source_path, hwts_output_filename)
-            result = hwtslog_parser.execute()
-            if not result:
-                logger.error("Profiling: fail to parse hwts log file.")
-                return
+            _ = hwtslog_parser.execute()
 
             # parse Framework file, and get the relation of op and tasks
             framework_parser = FrameworkParser(job_id, self._dev_id, self._output_path)
@@ -169,6 +168,7 @@ class Profiler:
             # get op compute time from hwts data and framework data, write output_op_compute_time.txt
             opcompute_output_filename = self._opcompute_output_filename_target + self._dev_id + ".txt"
             opcompute_output_filename = os.path.join(self._output_path, opcompute_output_filename)
+            opcompute_output_filename = validate_and_normalize_path(opcompute_output_filename)
             optime_parser = OPComputeTimeParser(
                 hwts_output_filename, opcompute_output_filename,
                 op_task_dict, self._output_path, self._dev_id
@@ -178,6 +178,7 @@ class Profiler:
             # parse DATA_PREPROCESS.dev.AICPU file, write output_data_preprocess_aicpu_x.txt
             output_data_preprocess_aicpu = self._aicpu_op_output_filename_target + self._dev_id + ".txt"
             output_data_preprocess_aicpu = os.path.join(self._output_path, output_data_preprocess_aicpu)
+            output_data_preprocess_aicpu = validate_and_normalize_path(output_data_preprocess_aicpu)
             aicpu_data_parser = DataPreProcessParser(source_path, output_data_preprocess_aicpu)
             aicpu_data_parser.execute()
 
@@ -230,10 +231,13 @@ class Profiler:
             self._output_path,
             'step_trace_point_info.json'
         )
+        step_trace_intermediate_file_path = validate_and_normalize_path(step_trace_intermediate_file_path)
+        point_info_file_path = validate_and_normalize_path(point_info_file_path)
         # whether keep the first step
         skip_first_step_flag = framework_parser.check_op_name(INIT_OP_NAME)
         point_info = framework_parser.point_info
         # parser the step trace files and save the result to disk
+        source_path = validate_and_normalize_path(source_path)
         parser = StepTraceParser(input_dir=source_path,
                                  output_file_path=step_trace_intermediate_file_path,
                                  job_id=self._job_id_env,