!6582 [gpu timeline] fix a bug of gpu timeline multi card

Merge pull request !6582 from gzhcv/GpuTimeline

mindspore/profiler/profiling.py

@@ -287,10 +287,18 @@ class Profiler:
         """Used for gpu, generate timeline info, write to json format file."""
         try:
             size_limit = 100 * 1024 * 1024  # 100MB
-            timeline_generator = GpuTimelineGenerator(self._output_path, self._dev_id)
-            timeline_generator.init_timeline()
-            timeline_generator.write_timeline(size_limit)
-            timeline_generator.write_timeline_summary()
+            #stastic the number of dev_id
+            file_list = os.listdir(self._output_path)
+            dev_id_list = []
+            for file_name in file_list:
+                if file_name.startswith('gpu_op_detail'):
+                    _dev_id = file_name.split('.')[0].split('_')[-1]
+                    dev_id_list.append(_dev_id)
+            for dev_id in dev_id_list:
+                timeline_generator = GpuTimelineGenerator(self._output_path, dev_id)
+                timeline_generator.init_timeline()
+                timeline_generator.write_timeline(size_limit)
+                timeline_generator.write_timeline_summary()
         except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
             logger.warning('Fail to write timeline data: %s', err)
             raise RuntimeError('Fail to write timeline data.')