add some test case for profiler

5 years ago · 54f2cf4ef0
--- a/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0
@@ -0,0 +1,4 @@
 op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
 op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
 op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
 op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"
--- a/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0.done
+++ b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0.done
--- a/tests/ut/data/profiler_data/JOB1/Framework.host.vm.task_desc_info.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.task_desc_info.0.slice_0
@@ -0,0 +1,4 @@
 Default/Cast-op6 32 51517 0
 Default/TransData-op7 32 51518 0
 Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
 Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0
--- a/tests/ut/data/profiler_data/JOB1/pipeline_profiling_0.json
+++ b/tests/ut/data/profiler_data/JOB1/pipeline_profiling_0.json
@@ -0,0 +1,55 @@
 {
  "sampling_interval": 10,
  "op_info": [
    {
      "op_id": 4,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": null,
      "children": [3]
    },
    {
      "op_id": 3,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": null
    },
    {
      "op_id": 2,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": null
    },
    {
      "op_id": 1,
      "op_type": "Shuffle",
      "num_workers": 1,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": [2, 4]
    },
    {
      "op_id": 0,
      "op_type": "Batch",
      "num_workers": 4,
      "metrics": null,
      "children": [1]
    }
  ]
 }
--- a/tests/ut/data/profiler_data/JOB1/training_trace.46.dev.profiler_default_tag.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB1/training_trace.46.dev.profiler_default_tag.0.slice_0
--- a/tests/ut/data/profiler_data/JOB2/pipeline_profiling_0.json
+++ b/tests/ut/data/profiler_data/JOB2/pipeline_profiling_0.json
@@ -0,0 +1,48 @@
 {
  "sampling_interval": 10,
  "op_info": [
    {
      "op_id": 3,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": null
    },
    {
      "op_id": 2,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": null
    },
    {
      "op_id": 1,
      "op_type": "Shuffle",
      "num_workers": 1,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": [2, 3]
    },
    {
      "op_id": 0,
      "op_type": "Batch",
      "num_workers": 4,
      "metrics": null,
      "children": [1]
    }
  ]
 }
--- a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.graph_desc_info.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.graph_desc_info.0.slice_0
@@ -0,0 +1,4 @@
 op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
 op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
 op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
 op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"
--- a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.point.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.point.0.slice_0
@@ -0,0 +1,2 @@
 1 Default/Cast-op6
 2 Default/TransData-op7
--- a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.task_desc_info.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.task_desc_info.0.slice_0
@@ -0,0 +1,4 @@
 Default/Cast-op6 32 1 0
 Default/TransData-op7 32 2 0
 Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 3 0
 Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 4 0
--- a/tests/ut/data/profiler_data/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0
+++ b/tests/ut/data/profiler_data/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0
--- a/tests/ut/data/profiler_data/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt
+++ b/tests/ut/data/profiler_data/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt
@@ -0,0 +1,5 @@
 serial_number node_type_name total_time(ms) dispatch_time(ms) run_start run_end
 1 InitData 1.567 0.1 2298200409 2298200538
 2 GetNext 0.989 0.087 2302769932 2302769980
 3 TruncatedNormal 1.566 0.105 4098200409 4098200538
 AI CPU Total Time(ms): 4.122000
--- a/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.graph_desc_info.0.JOB2.slice_0
+++ b/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.graph_desc_info.0.JOB2.slice_0
@@ -0,0 +1,4 @@
 op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
 op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
 op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
 op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"
--- a/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.task_desc_info.0.JOB2.slice_0
+++ b/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.task_desc_info.0.JOB2.slice_0
@@ -0,0 +1,4 @@
 Default/Cast-op6 32 51517 0
 Default/TransData-op7 32 51518 0
 Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
 Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0
--- a/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_detail.csv
+++ b/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_detail.csv
@@ -0,0 +1,11 @@
 full_op_time,execution_time
 Default/AtomicAddrClean-op104,0.00133
 Default/AtomicAddrClean-op105,0.000987
 Default/AtomicAddrClean-op106,0.001129
 Default/Cast-op10,0.00466
 Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,0.002366
 Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,0.004879
 Default/TransData-op11,0.006366
 Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,0.006782
 Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,0.05651
 Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,0.370864
--- a/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_type.csv
+++ b/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_type.csv
@@ -0,0 +1,6 @@
 op_type,execution_time,execution_frequency,percent
 AtomicAddrClean,0.007283,6,0.49
 Cast,0.053395,13,3.63
 TransData,0.121800,5,8.23
 Conv2D,0.063656,2,4.33
 MatMul,1.085982,9,73.80
--- a/tests/ut/data/profiler_data/profiler/framework_raw_0.csv
+++ b/tests/ut/data/profiler_data/profiler/framework_raw_0.csv
@@ -0,0 +1,5 @@
 task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info
 51517,0,32,Default/Cast-op6,Cast-op6,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}}"
 51518,0,32,Default/TransData-op7,TransData-op7,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,224,224,16""}}"
 51519,0,32,Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5,Cast-op5,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""49,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""49,4,16,16""}}"
 51522,0,4,Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28,Cast-op28,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""4,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""4,4,16,16""}}"
--- a/tests/ut/data/profiler_data/profiler/framework_raw_1.csv
+++ b/tests/ut/data/profiler_data/profiler/framework_raw_1.csv
@@ -0,0 +1,11 @@
 task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info
 30290,0,1,Default/AtomicAddrClean-op104,AtomicAddrClean-op104,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": """"}}"
 30295,0,1,Default/AtomicAddrClean-op105,AtomicAddrClean-op105,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""10""}}"
 30300,0,1,Default/AtomicAddrClean-op106,AtomicAddrClean-op106,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}}"
 30268,0,32,Default/Cast-op10,Cast-op10,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}"
 30271,0,9,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,Cast-op12,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}}"
 30320,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,Cast-op53,Cast,Gradients,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,28,28,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}"
 30269,0,32,Default/TransData-op11,TransData-op11,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}"
 30308,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,TransData-op44,TransData,Gradients,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,16,5,5""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,5,5,16""}}"
 30272,0,32,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,Conv2D-op13,Conv2D,Default,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32,16""}, ""input_1"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}"
 30286,0,1,Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,MatMul-op9,MatMul,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,120""}, ""input_1"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84,120""}, ""input_2"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,84""}}"
--- a/tests/ut/data/profiler_data/profiler/minddata_pipeline_raw_0.csv
+++ b/tests/ut/data/profiler_data/profiler/minddata_pipeline_raw_0.csv
@@ -0,0 +1,5 @@
 op_id,op_type,num_workers,output_queue_size,output_queue_average_size,output_queue_length,output_queue_usage_rate,sample_interval,parent_id,children_id
 0,Batch,4,,,,,10,,[1]
 1,Shuffle,1,"[10, 20, 30]",20.0,64,0.3125,10,0,"[2, 3]"
 2,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1,
 3,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1,
--- a/tests/ut/data/profiler_data/profiler/pipeline_profiling_1.json
+++ b/tests/ut/data/profiler_data/profiler/pipeline_profiling_1.json
@@ -0,0 +1,55 @@
 {
  "sampling_interval": 10,
  "op_info": [
    {
      "op_id": 4,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": null,
      "children": [3]
    },
    {
      "op_id": 3,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": null
    },
    {
      "op_id": 2,
      "op_type": "TFReader",
      "num_workers": 4,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": null
    },
    {
      "op_id": 1,
      "op_type": "Shuffle",
      "num_workers": 1,
      "metrics": {
        "output_queue": {
          "size": [10, 20, 30],
          "length": 64
        }
      },
      "children": [2, 4]
    },
    {
      "op_id": 0,
      "op_type": "Batch",
      "num_workers": 4,
      "metrics": null,
      "children": [1]
    }
  ]
 }
--- a/tests/ut/data/profiler_data/profiler/step_trace_raw_0_detail_time.csv
+++ b/tests/ut/data/profiler_data/profiler/step_trace_raw_0_detail_time.csv
@@ -0,0 +1,22 @@
 step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0
 1,45000030081,45004033128,4003047,45000030081,45001733025,0,1702944,2300103,45000042679,45000060275,17596,45001048152,45001346254,298102,45002247411,45002448354,200943,45000049687,45000075987,26300
 2,45004033128,45017085658,13052530,45013070937,45014785314,9037809,1714377,2300344,45013085379,45013105429,20050,45014087119,45014385136,298017,45015297166,45015504449,207283,45013084925,45013118334,33409
 3,45017085658,45030119392,13033734,45026116231,45027818443,9030573,1702212,2300949,45026131909,45026150554,18645,45027134392,45027430418,296026,45028337093,45028537767,200674,45026129217,45026160937,31720
 4,45030119392,45043158607,13039215,45039152348,45040856975,9032956,1704627,2301632,45039169890,45039188966,19076,45040169338,45040466770,297432,45041374122,45041567754,193632,45039171681,45039193865,22184
 5,45043158607,45056198128,13039521,45052190932,45053898028,9032325,1707096,2300100,45052207675,45052222642,14967,45053204442,45053505540,301098,45054413207,45054616536,203329,45052201931,45052237599,35668
 6,45056198128,45069239564,13041436,45065233106,45066939463,9034978,1706357,2300101,45065245482,45065272534,27052,45066248423,45066546419,297996,45067455113,45067659145,204032,45065245817,45065279896,34079
 7,45069239564,45082281383,13041819,45078274997,45079980193,9035433,1705196,2301190,45078293910,45078312935,19025,45079287754,45079593841,306087,45080492957,45080691395,198438,45078292067,45078322277,30210
 8,45082281383,45095336378,13054995,45091321488,45093036084,9040105,1714596,2300294,45091338628,45091359138,20510,45092338469,45092638994,300525,45093554195,45093747470,193275,45091341356,45091369667,28311
 9,45095336378,45108372225,13035847,45104363079,45106071009,9026701,1707930,2301216,45104374524,45104400088,25564,45105378751,45105683029,304278,45106587481,45106785336,197855,45104382131,45104410852,28721
 10,45108372225,45121412413,13040188,45117401873,45119111301,9029648,1709428,2301112,45117417721,45117439668,21947,45118413083,45118718050,304967,45119629347,45119829996,200649,45117421502,45117446718,25216
 11,45121412413,45134477662,13065249,45130459598,45132175723,9047185,1716125,2301939,45130478168,45130498936,20768,45131477957,45131775220,297263,45132691645,45132893707,202062,45130470285,45130501652,31367
 12,45134477662,45147533298,13055636,45143521860,45145232553,9044198,1710693,2300745,45143533787,45143557293,23506,45144533554,45144841545,307991,45145744997,45145952255,207258,45143537383,45143563466,26083
 13,45147533298,45160588134,13054836,45156570201,45158286694,9036903,1716493,2301440,45156581069,45156609506,28437,45157581617,45157880841,299224,45158806166,45158999875,193709,45156589050,45156615664,26614
 14,45160588134,45173640064,13051930,45169625906,45171339426,9037772,1713520,2300638,45169637432,45169661754,24322,45170639482,45170940949,301467,45171853721,45172056606,202885,45169644605,45169673410,28805
 15,45173640064,45186671634,13031570,45182666696,45184371430,9026632,1704734,2300204,45182678355,45182698471,20116,45183679568,45183981082,301514,45184887156,45185083035,195879,45182680062,45182708455,28393
 16,45186671634,45199720448,13048814,45195714716,45197420410,9043082,1705694,2300038,45195728993,45195754646,25653,45196732493,45197028048,295555,45197934921,45198139237,204316,45195733069,45195764102,31033
 17,45199720448,45212762605,13042157,45208758416,45210460864,9037968,1702448,2301741,45208771010,45208790367,19357,45209773548,45210074988,301440,45210978277,45211173577,195300,45208773143,45208803280,30137
 18,45212762605,45225814601,13051996,45221801814,45223514580,9039209,1712766,2300021,45221815911,45221839644,23733,45222819211,45223114544,295333,45224031469,45224234043,202574,45221812106,45221849103,36997
 19,45225814601,45238848430,13033829,45234842015,45236548356,9027414,1706341,2300074,45234855444,45234876469,21025,45235853358,45236160825,307467,45237063061,45237260964,197903,45234857141,45234882976,25835
 20,45238848430,45251899738,13051308,45247879385,45249598280,9030955,1718895,2301458,45247896725,45247917316,20591,45248896361,45249193681,297320,45250117916,45250315651,197735,45247894228,45247926723,32495
 -,45121436513,45134482124,13045611,45130471874,45132181322,9035360,1709449,2300802,45130486422,45130508229,21808,45131486785,45131787364,300579,45132697369,45132897305,199936,45130487458,45130517315,29857
--- a/tests/ut/data/profiler_data/profiler/step_trace_raw_10_detail_time.csv
+++ b/tests/ut/data/profiler_data/profiler/step_trace_raw_10_detail_time.csv
@@ -0,0 +1,42 @@
 step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0
 1,45000025226,45004034753,4009527,45000025226,45001734362,0,1709136,2300391,45000044023,45000060886,16863,45001043581,45001343373,299792,45002254048,45002452830,198782,45000043807,45000065736,21929
 2,45004034753,45017091420,13056667,45013073790,45014789509,9039037,1715719,2301911,45013085205,45013104210,19005,45014086339,45014393261,306922,45015299546,45015501808,202262,45013085040,45013119810,34770
 3,45017091420,45030144372,13052952,45026123867,45027843651,9032447,1719784,2300721,45026138546,45026154524,15978,45027135742,45027437486,301744,45028363120,45028560901,197781,45026136046,45026171363,35317
 4,45030144372,45043184486,13040114,45039173149,45040883087,9028777,1709938,2301399,45039190927,45039209948,19021,45040185915,45040484897,298982,45041399754,45041594775,195021,45039192768,45039221423,28655
 5,45043184486,45056241064,13056578,45052223555,45053940709,9039069,1717154,2300355,45052241736,45052262186,20450,45053239605,45053540866,301261,45054452604,45054654505,201901,45052233932,45052265774,31842
 6,45056241064,45069291346,13050282,45065278144,45066991121,9037080,1712977,2300225,45065293660,45065316136,22476,45066289480,45066589910,300430,45067511002,45067701731,190729,45065293679,45065321296,27617
 7,45069291346,45082344927,13053581,45078335376,45080043268,9044030,1707892,2301659,45078353164,45078365382,12218,45079354748,45079648384,293636,45080557453,45080760374,202921,45078353030,45078384530,31500
 8,45082344927,45095382554,13037627,45091368697,45093080797,9023770,1712100,2301757,45091381244,45091405208,23964,45092382630,45092684285,301655,45093590961,45093796698,205737,45091381199,45091413840,32641
 9,45095382554,45108433947,13051393,45104419947,45106132133,9037393,1712186,2301814,45104432587,45104457476,24889,45105431458,45105735476,304018,45106651213,45106845305,194092,45104435207,45104466677,31470
 10,45108433947,45121486591,13052644,45117469353,45119185969,9035406,1716616,2300622,45117483627,45117504869,21242,45118483411,45118788540,305129,45119696660,45119898575,201915,45117485587,45117510985,25398
 11,45121486591,45134546571,13059980,45130528618,45132244809,9042027,1716191,2301762,45130539730,45130561122,21392,45131538695,45131846715,308020,45132759789,45132960848,201059,45130545378,45130569412,24034
 12,45134546571,45147608222,13061651,45143597023,45145307273,9050452,1710250,2300949,45143615771,45143631460,15689,45144610592,45144910736,300144,45145818642,45146024326,205684,45143613528,45143640223,26695
 13,45147608222,45160663790,13055568,45156648696,45158362923,9040474,1714227,2300867,45156663193,45156685466,22273,45157661576,45157963074,301498,45158881212,45159074431,193219,45156667038,45156694912,27874
 14,45160663790,45173707626,13043836,45169694535,45171407246,9030745,1712711,2300380,45169710667,45169727936,17269,45170705802,45171013806,308004,45171924100,45172120273,196173,45169708524,45169739038,30514
 15,45173707626,45186754860,13047234,45182750254,45184454036,9042628,1703782,2300824,45182765445,45182789799,24354,45183761335,45184065169,303834,45184973312,45185170444,197132,45182769451,45182799598,30147
 16,45186754860,45199798718,13043858,45195792271,45197497908,9037411,1705637,2300810,45195804771,45195827915,23144,45196804016,45197108243,304227,45198013357,45198209858,196501,45195806656,45195841674,35018
 17,45199798718,45212854993,13056275,45208834355,45210553378,9035637,1719023,2301615,45208850179,45208865588,15409,45209851018,45210151436,300418,45211073169,45211271792,198623,45208847052,45208876998,29946
 18,45212854993,45225893712,13038719,45221888939,45223593704,9033946,1704765,2300008,45221901732,45221924983,23251,45222908795,45223203590,294795,45224105803,45224313354,207551,45221899792,45221938802,39010
 19,45225893712,45238941242,13047530,45234926295,45236640454,9032583,1714159,2300788,45234938628,45234957237,18609,45235942710,45236239983,297273,45237159532,45237356140,196608,45234938330,45234976170,37840
 20,45238941242,45251979177,13037935,45247977674,45249678116,9036432,1700442,2301061,45247990919,45248013476,22557,45248991451,45249294742,303291,45250195733,45250395760,200027,45247988950,45248024969,36019
 21,45251979177,45265018752,13039575,45261005416,45262718472,9026239,1713056,2300280,0,0,0,0,0,0,0,0,0,0,0,0
 22,45265018752,45278062782,13044030,45274047185,45275762095,9028433,1714910,2300687,0,0,0,0,0,0,0,0,0,0,0,0
 23,45278062782,45291105708,13042926,45287094000,45288805223,9031218,1711223,2300485,0,0,0,0,0,0,0,0,0,0,0,0
 24,45291105708,45304155918,13050210,45300150844,45301854040,9045136,1703196,2301878,0,0,0,0,0,0,0,0,0,0,0,0
 25,45304155918,45317206695,13050777,45313191948,45314905714,9036030,1713766,2300981,0,0,0,0,0,0,0,0,0,0,0,0
 26,45317206695,45330265105,13058410,45326256021,45327964581,9049326,1708560,2300524,0,0,0,0,0,0,0,0,0,0,0,0
 27,45330265105,45343324012,13058907,45339305124,45341023739,9040019,1718615,2300273,0,0,0,0,0,0,0,0,0,0,0,0
 28,45343324012,45356374571,13050559,45352366211,45354073401,9042199,1707190,2301170,0,0,0,0,0,0,0,0,0,0,0,0
 29,45356374571,45369429514,13054943,45365417827,45367128283,9043256,1710456,2301231,0,0,0,0,0,0,0,0,0,0,0,0
 30,45369429514,45382479199,13049685,45378476397,45380177297,9046883,1700900,2301902,0,0,0,0,0,0,0,0,0,0,0,0
 31,45382479199,45395530376,13051177,45391510137,45393229377,9030938,1719240,2300999,0,0,0,0,0,0,0,0,0,0,0,0
 32,45395530376,45408571765,13041389,45404559082,45406270720,9028706,1711638,2301045,0,0,0,0,0,0,0,0,0,0,0,0
 33,45408571765,45421635175,13063410,45417619223,45419334221,9047458,1714998,2300954,0,0,0,0,0,0,0,0,0,0,0,0
 34,45421635175,45434672219,13037044,45430669445,45432371312,9034270,1701867,2300907,0,0,0,0,0,0,0,0,0,0,0,0
 35,45434672219,45447714036,13041817,45443704548,45445413852,9032329,1709304,2300184,0,0,0,0,0,0,0,0,0,0,0,0
 36,45447714036,45460765153,13051117,45456753675,45458463701,9039639,1710026,2301452,0,0,0,0,0,0,0,0,0,0,0,0
 37,45460765153,45473829105,13063952,45469808281,45471527400,9043128,1719119,2301705,0,0,0,0,0,0,0,0,0,0,0,0
 38,45473829105,45486884190,13055085,45482867237,45484583534,9038132,1716297,2300656,0,0,0,0,0,0,0,0,0,0,0,0
 39,45486884190,45499928571,13044381,45495917628,45497627921,9033438,1710293,2300650,0,0,0,0,0,0,0,0,0,0,0,0
 40,45499928571,45512973815,13045244,45508968990,45510673699,9040419,1704709,2300116,0,0,0,0,0,0,0,0,0,0,0,0
 -,45251983006,45265032725,13049720,45261020353,45262731761,9037347,1711408,2300964,21986676455,21986686280,9825,21987163213,21987310272,147058,21987754537,21987851587,97050,21986676441,21986691731,15290
--- a/tests/ut/python/profiler/init.py
+++ b/tests/ut/python/profiler/init.py
@@ -0,0 +1,21 @@
 # Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
 """Unit test for profiler."""
 import os

 RAW_DATA_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../data/profiler_data'))
 RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB1'))
 RAW_DATA_JOB2 = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB2'))
 PROFILER_DIR = os.path.realpath(os.path.join(RAW_DATA_BASE, 'profiler'))
--- a/tests/ut/python/profiler/parser/init.py
+++ b/tests/ut/python/profiler/parser/init.py
@@ -0,0 +1,14 @@
 # Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
--- a/tests/ut/python/profiler/parser/test_aicpu_parser.py
+++ b/tests/ut/python/profiler/parser/test_aicpu_parser.py
@@ -0,0 +1,74 @@
 # Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
 """Test the aicpu parser."""
 import os
 import tempfile
 import shutil

 from unittest import TestCase

 from mindspore.profiler.parser.aicpu_data_parser import DataPreProcessParser


 def get_result(file_path):
    """
    Get result from the aicpu file.

    Args:
        file_path (str): The aicpu file path.

    Returns:
        list[list], the parsed aicpu information.
    """
    result = []
    try:
        file = open(file_path, 'r')
        result.append(file.read())
        return result
    finally:
        if file:
            file.close()


 class TestAicpuParser(TestCase):
    """Test the class of Aicpu Parser."""

    def setUp(self) -> None:
        """Initialization before test case execution."""
        self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
                                                           '../../../data/profiler_data/'
                                                           'JOB_AICPU/data'))
        self.expect_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
                                                        '../../../data/profiler_data/'
                                                        'JOB_AICPU/expect'))
        self.output_path = tempfile.mkdtemp(prefix='output_data_preprocess_aicpu_')
        self.output_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt')
        self.expect_file = os.path.join(self.expect_dir, 'output_data_preprocess_aicpu_0.txt')

    def test_aicpu_parser(self):
        """Test the class of Aicpu Parser."""
        data = DataPreProcessParser(self.profiling_dir, self.output_file)
        data.execute()
        expect_result = get_result(self.expect_file)
        result = get_result(self.output_file)
        shutil.rmtree(self.output_path)
        assert expect_result == result

    def test_aicpu_parser_file_not_exist(self):
        """Test the class of Aicpu Parser."""
        profiling_dir = os.path.realpath(os.path.join(self.profiling_dir, 'data'))
        data = DataPreProcessParser(profiling_dir, self.output_file)
        data.execute()
        shutil.rmtree(self.output_path)
--- a/tests/ut/python/profiler/parser/test_framework_parser.py
+++ b/tests/ut/python/profiler/parser/test_framework_parser.py
@@ -0,0 +1,128 @@
 # Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
 """Test the framework parser module."""
 import csv
 import os
 import shutil
 import tempfile
 from unittest import mock

 import pytest

 from mindspore.profiler.common.exceptions.exceptions import \
    ProfilerFileNotFoundException
 from mindspore.profiler.parser.framework_parser import FrameworkParser
 from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA_BASE


 def get_framework_result(file_path):
    """
    Get framework result from the framework file.

    Args:
        file_path (str): The framework file path.

    Returns:
        list[list], the parsed framework information.
    """
    result = []
    with open(file_path, 'r') as file:
        csv_reader = csv.reader(file)
        for row in csv_reader:
            result.append(row)
    return result


 class TestFrameworkParser:
    """Test the class of `FrameworkParser`."""
    def setup_method(self):
        """Initialization before test case execution."""
        with mock.patch.object(FrameworkParser, '_raw_data_dir', RAW_DATA_BASE):
            self._output_path_1 = tempfile.mkdtemp(prefix='test_framework_parser_')
            self._parser_1 = FrameworkParser('JOB1', '0', self._output_path_1)
            self._output_path_2 = tempfile.mkdtemp(prefix='test_framework_parser_')
            self._parser_2 = FrameworkParser('JOB2', '0', self._output_path_2)
            self._output_path_4 = tempfile.mkdtemp(prefix='test_framework_parser_')
            self._parser_4 = FrameworkParser('JOB4', '0', self._output_path_4)

    def teardown_method(self) -> None:
        """Clear up after test case execution."""
        shutil.rmtree(self._output_path_1)
        shutil.rmtree(self._output_path_2)
        shutil.rmtree(self._output_path_4)

    def test_save_path(self):
        """Test the querying save path function."""
        expect_result = os.path.join(self._output_path_1, 'framework_raw_0.csv')
        assert expect_result == self._parser_1.save_path

        expect_result = os.path.join(self._output_path_2, 'framework_raw_0.csv')
        assert expect_result == self._parser_2.save_path

    def test_point_info(self):
        """Test the querying point info function."""
        expect_result = {
            1: 'Default/Cast-op6',
            2: 'Default/TransData-op7'
        }
        assert expect_result == self._parser_4.point_info

    def test_to_task_id_full_op_name_dict(self):
        """Test the querying task id and full operator name dict function."""
        expect_result = {
            '51517': 'Default/Cast-op6',
            '51518': 'Default/TransData-op7',
            '51519': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
            '51522': 'Default/network-WithLossCell/_backbone-ResNet/'
                     'layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28'
        }
        assert expect_result == self._parser_1.to_task_id_full_op_name_dict()
        assert expect_result == self._parser_2.to_task_id_full_op_name_dict()

        expect_result = {
            '0_1': 'Default/Cast-op6',
            '0_2': 'Default/TransData-op7',
            '0_3': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
            '0_4': 'Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/'
                   '0-ResidualBlock/conv1-Conv2d/Cast-op28'
        }
        assert expect_result == self._parser_4.to_task_id_full_op_name_dict()

    def test_parse(self):
        """Test the parse function."""
        expect_framework_file = os.path.join(PROFILER_DIR, 'framework_raw_0.csv')
        expect_framework_file = os.path.realpath(expect_framework_file)
        expect_result = get_framework_result(expect_framework_file)

        self._parser_1.parse()
        framework_file = os.path.join(self._output_path_1, 'framework_raw_0.csv')
        result = get_framework_result(framework_file)
        assert expect_result == result

        self._parser_2.parse()
        framework_file = os.path.join(self._output_path_2, 'framework_raw_0.csv')
        result = get_framework_result(framework_file)
        assert expect_result == result

    @mock.patch('os.listdir')
    @mock.patch('os.path.isdir')
    def test_create_framework_parser_fail_1(self, *args):
        """Test the function of fail to create framework parser."""
        args[0].return_value = True
        args[1].return_value = []
        with pytest.raises(ProfilerFileNotFoundException) as exc_info:
            FrameworkParser('JOB1', '0')
        assert exc_info.value.error_code == '50546084'
        assert exc_info.value.message == 'The file <Framework> not found.'
--- a/tests/ut/python/profiler/parser/test_minddata_pipeline_parser.py
+++ b/tests/ut/python/profiler/parser/test_minddata_pipeline_parser.py
@@ -0,0 +1,93 @@
 # Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
 """Test the minddata pipeline parser module."""
 import csv
 import os
 import shutil
 import tempfile

 from mindspore.profiler.parser.minddata_pipeline_parser import \
    MinddataPipelineParser
 from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA, RAW_DATA_JOB2


 def get_minddata_pipeline_result(file_path):
    """
    Get minddata pipeline result from the minddata pipeline file.

    Args:
        file_path (str): The minddata pipeline file path.

    Returns:
        list[list], the parsed minddata pipeline information.
    """
    result = []
    with open(file_path, 'r') as file:
        csv_reader = csv.reader(file)
        for row in csv_reader:
            result.append(row)
    return result


 class TestMinddataPipelineParser:
    """Test the class of `MinddataPipelineParser`."""
    def setup_method(self):
        """Initialization before test case execution."""
        self._output_path_1 = tempfile.mkdtemp(
            prefix='test_minddata_pipeline_parser_'
        )
        self._parser_1 = MinddataPipelineParser(
            RAW_DATA, '0', self._output_path_1
        )

        self._output_path_2 = tempfile.mkdtemp(
            prefix='test_minddata_pipeline_parser_'
        )
        self._parser_2 = MinddataPipelineParser(
            RAW_DATA_JOB2, '0', self._output_path_2
        )

    def teardown_method(self) -> None:
        """Clear up after test case execution."""
        shutil.rmtree(self._output_path_1)
        shutil.rmtree(self._output_path_2)

    def test_save_path(self):
        """Test the querying save path function."""
        expect_result = os.path.join(
            self._output_path_1, 'minddata_pipeline_raw_0.csv'
        )
        assert expect_result == self._parser_1.save_path

    def test_parse(self):
        """Test the parse function."""
        expect_pipeline_file = os.path.join(
            PROFILER_DIR, 'minddata_pipeline_raw_0.csv'
        )
        expect_result = get_minddata_pipeline_result(expect_pipeline_file)

        self._parser_1.parse()
        pipeline_file = os.path.join(
            self._output_path_1, 'minddata_pipeline_raw_0.csv'
        )
        result = get_minddata_pipeline_result(pipeline_file)
        assert expect_result == result

        self._parser_2.parse()
        pipeline_file = os.path.join(
            self._output_path_2, 'minddata_pipeline_raw_0.csv'
        )
        result = get_minddata_pipeline_result(pipeline_file)
        assert expect_result == result