zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

!28427 Clean PyNative Profiling code. 

Merge pull request !28427 from caifubi/master-pynative-codeclean
tags/v1.6.0
i-robot Gitee 4 years ago
parent
e91f2286a1 ee3b82ac09
commit
afaef45d64
5 changed files with 7 additions and 304 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      mindspore/ccsrc/pipeline/pynative/CMakeLists.txt
  2. +0
    -233
      mindspore/ccsrc/pipeline/pynative/pynative_profiling.cc
  3. +0
    -56
      mindspore/ccsrc/pipeline/pynative/pynative_profiling.h
  4. +4
    -9
      mindspore/ccsrc/runtime/device/kernel_runtime.cc
  5. +2
    -5
      mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc

+ 1
- 1
mindspore/ccsrc/pipeline/pynative/CMakeLists.txt View File

@@ -1,4 +1,4 @@
file(GLOB_RECURSE _PYNATIVE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "pynative_execute.cc" "pynative_profiling.cc")
file(GLOB_RECURSE _PYNATIVE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "pynative_execute.cc")

if(ENABLE_GE)
file(GLOB_RECURSE _GE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "pynative_execute_ge.cc")


+ 0
- 233
mindspore/ccsrc/pipeline/pynative/pynative_profiling.cc View File

@@ -1,233 +0,0 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "pipeline/pynative/pynative_profiling.h"
#include "utils/profile.h"
#include "utils/ms_context.h"
#include "utils/utils.h"
#include "profiler/device/profiling.h"

namespace mindspore {
constexpr int kDeviceInfoCoutWidth = 25;
constexpr int kHostTimePointCoutWidth = 35;
constexpr int kHostTimeCoutWidth = 30;

void PynativeProfiler::SetEnableProfilingFlag() {
#ifndef ENABLE_SECURITY
static bool flag = false;
if (flag) {
return;
}
auto profiler_manager = profiler::ProfilerManager::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_manager);
enable_profiler_flag_ = profiler_manager->GetProfilingEnableFlag();
flag = true;
#endif
}

void PynativeProfiler::Reset() {
stage_time_point_vec_.clear();
stage_stat_time_vec_.clear();
op_name_launch_time_point_vec_.clear();
op_name_launch_time_vec_.clear();
}

void PynativeProfiler::SetDeviceOpNameAndLaunchTimePoint(
const std::pair<std::string, std::pair<double, double>> &name_start_end) {
if (!enable_profiler_flag_ || op_name_launch_time_point_vec_.size() > kMaxVectorSize) {
return;
}
op_name_launch_time_point_vec_.push_back(name_start_end);
}

void PynativeProfiler::SetDeviceOpNameAndLaunchCostTime(const std::pair<std::string, double> &name_time) {
if (!enable_profiler_flag_ || op_name_launch_time_vec_.size() > kMaxVectorSize) {
return;
}
op_name_launch_time_vec_.push_back(name_time);
}

void PynativeProfiler::ExportDeviceInfoToFile() {
MS_LOG(DEBUG) << "Op name launch time point vec size: " << op_name_launch_time_point_vec_.size();
if (!enable_profiler_flag_ || op_name_launch_time_point_vec_.empty()) {
return;
}
static std::ofstream of_device("device_profiling_data.csv", std::ios::app);
of_device.setf(std::ios::fixed, std::ios::floatfield);
of_device << "DeviceIndex" << ',' << "op_name" << ',' << "LaunchStartTime(s)" << ',' << "LaunchEndTime(s)" << ','
<< "LaunchCostTime(ms)" << std::endl;
if (op_name_launch_time_point_vec_.size() != op_name_launch_time_vec_.size()) {
MS_LOG(EXCEPTION) << "The size of the two vector is not equal, the two vector size is "
<< op_name_launch_time_point_vec_.size() << " and " << op_name_launch_time_vec_.size();
}
for (size_t i = 1; i <= op_name_launch_time_point_vec_.size(); ++i) {
of_device << i << ',' << op_name_launch_time_point_vec_[i - 1].first << ','
<< op_name_launch_time_point_vec_[i - 1].second.first << ','
<< op_name_launch_time_point_vec_[i - 1].second.second << ','
<< op_name_launch_time_vec_[i - 1].second * kBasicTimeTransferUnit << std::endl;
}
op_name_launch_time_point_vec_.clear();
op_name_launch_time_vec_.clear();
}

void PynativeProfiler::ExportDeviceInfoToScreen() {
MS_LOG(DEBUG) << "Op name launch time point vec size: " << op_name_launch_time_point_vec_.size();
if (!enable_profiler_flag_ || op_name_launch_time_point_vec_.empty()) {
return;
}
if (op_name_launch_time_point_vec_.size() != op_name_launch_time_vec_.size()) {
MS_LOG(EXCEPTION) << "The size of the two vector is not equal, the two vector size is "
<< op_name_launch_time_point_vec_.size() << " and " << op_name_launch_time_vec_.size();
}
std::cout << "====================================DeviceInfo===================================" << std::endl;
std::vector<std::string> head_str = {"DeviceIndex", "op_name", "LaunchStartTime(s)", "LaunchEndTime(s)",
"LaunchCostTime(ms)"};
std::cout.setf(std::ios::fixed, std::ios::floatfield);
std::cout.setf(std::ios::left);
for (const auto &str : head_str) {
std::cout.width(kDeviceInfoCoutWidth);
std::cout << str;
}
std::cout << std::endl;
for (size_t i = 1; i <= op_name_launch_time_point_vec_.size(); ++i) {
std::cout.width(kDeviceInfoCoutWidth);
std::cout << i;
std::cout.width(kDeviceInfoCoutWidth);
std::cout << op_name_launch_time_point_vec_[i - 1].first;
std::cout.width(kDeviceInfoCoutWidth);
std::cout << op_name_launch_time_point_vec_[i - 1].second.first;
std::cout.width(kDeviceInfoCoutWidth);
std::cout << op_name_launch_time_point_vec_[i - 1].second.second;
std::cout.width(kDeviceInfoCoutWidth);
std::cout << op_name_launch_time_vec_[i - 1].second * kBasicTimeTransferUnit;
std::cout << std::endl;
}
std::cout << "==============================================================================" << std::endl;
}

void PynativeProfiler::SetStageTimePoint(const std::string &stage_name, const std::string &flag) {
if (!enable_profiler_flag_ || stage_time_point_vec_.size() > kMaxVectorSize) {
return;
}
stage_time_point_vec_.emplace_back(stage_name, std::make_pair(flag, GetTime()));
}

double PynativeProfiler::SetStageTimePointWithReturn(const std::string &stage_name, const std::string &flag) {
if (!enable_profiler_flag_ || stage_time_point_vec_.size() > kMaxVectorSize) {
return 0;
}
double tmp_time = GetTime();
stage_time_point_vec_.emplace_back(stage_name, std::make_pair(flag, GetTime()));
return tmp_time;
}

void PynativeProfiler::SetStageStatTime(const std::string &stage_name, double cost_time) {
if (!enable_profiler_flag_ || stage_stat_time_vec_.size() > kMaxVectorSize) {
return;
}
stage_stat_time_vec_.emplace_back(stage_name, cost_time);
}

void PynativeProfiler::ExportStageTimePointToFile() {
if (!enable_profiler_flag_ || stage_time_point_vec_.empty()) {
return;
}
static std::ofstream of_host("host_stage_time_point_profiling_data.csv", std::ios::app);
of_host.setf(std::ios::fixed, std::ios::floatfield);
for (size_t i = 0; i < stage_time_point_vec_.size(); ++i) {
if (i == 0) {
of_host << stage_time_point_vec_[i].first + stage_time_point_vec_[i].second.first + "Time(s)";
continue;
}
of_host << ',' << stage_time_point_vec_[i].first + stage_time_point_vec_[i].second.first + "Time(s)";
}
of_host << std::endl;
for (size_t i = 0; i < stage_time_point_vec_.size(); ++i) {
if (i == 0) {
of_host << stage_time_point_vec_[i].second.second;
continue;
}
of_host << ',' << stage_time_point_vec_[i].second.second;
}
of_host << std::endl;
stage_time_point_vec_.clear();
}

void PynativeProfiler::ExportStageTimePointToScreen() {
if (!enable_profiler_flag_ || stage_time_point_vec_.empty()) {
return;
}
std::cout << "===============================StageTimePoint=================================" << std::endl;
std::cout.setf(std::ios::fixed, std::ios::floatfield);
std::cout.setf(std::ios::left);
for (const auto &i : stage_time_point_vec_) {
std::cout.width(kHostTimePointCoutWidth);
std::cout << i.first + i.second.first + "Time(s)";
}
std::cout << std::endl;
for (const auto &i : stage_time_point_vec_) {
std::cout.width(kHostTimePointCoutWidth);
std::cout << i.second.second * kBasicTimeTransferUnit;
}
std::cout << std::endl;
std::cout << "==============================================================================" << std::endl;
}

void PynativeProfiler::ExportStageStatTimeToFile() {
if (!enable_profiler_flag_ || stage_stat_time_vec_.empty()) {
return;
}
static std::ofstream of_host("host_stage_stat_time_profiling_data.csv", std::ios::app);
of_host.setf(std::ios::fixed, std::ios::floatfield);
for (size_t i = 0; i < stage_stat_time_vec_.size(); ++i) {
if (i == 0) {
of_host << stage_stat_time_vec_[i].first + "Time(ms)";
continue;
}
of_host << ',' << stage_stat_time_vec_[i].first + "Time(ms)";
}
of_host << std::endl;
for (size_t i = 0; i < stage_stat_time_vec_.size(); ++i) {
if (i == 0) {
of_host << stage_stat_time_vec_[i].second * kBasicTimeTransferUnit;
continue;
}
of_host << ',' << stage_stat_time_vec_[i].second * kBasicTimeTransferUnit;
}
of_host << std::endl;
stage_stat_time_vec_.clear();
}

void PynativeProfiler::ExportStageStatTimeToScreen() {
if (!enable_profiler_flag_ || stage_stat_time_vec_.empty()) {
return;
}
std::cout << "================================StageStatTime=================================" << std::endl;
std::cout.setf(std::ios::fixed, std::ios::floatfield);
std::cout.setf(std::ios::left);
for (const auto &i : stage_stat_time_vec_) {
std::cout.width(kHostTimeCoutWidth);
std::cout << i.first + "Time(ms)";
}
std::cout << std::endl;
for (const auto &i : stage_stat_time_vec_) {
std::cout.width(kHostTimeCoutWidth);
std::cout << i.second * kBasicTimeTransferUnit;
}
std::cout << std::endl;
std::cout << "==============================================================================" << std::endl;
}
} // namespace mindspore

+ 0
- 56
mindspore/ccsrc/pipeline/pynative/pynative_profiling.h View File

@@ -1,56 +0,0 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef MINDSPORE_CCSRC_PIPELINE_PYNATIVE_PYNATIVE_PROFILING_H_
#define MINDSPORE_CCSRC_PIPELINE_PYNATIVE_PYNATIVE_PROFILING_H_

#include <memory>
#include <string>
#include <vector>
#include <utility>

namespace mindspore {
class PynativeProfiler {
public:
PynativeProfiler(const PynativeProfiler &) = delete;
PynativeProfiler &operator=(const PynativeProfiler &) = delete;
static void SetEnableProfilingFlag();
static void Reset();
static void SetDeviceOpNameAndLaunchTimePoint(
const std::pair<std::string, std::pair<double, double>> &name_start_end);
static void SetDeviceOpNameAndLaunchCostTime(const std::pair<std::string, double> &name_time);
static void ExportDeviceInfoToFile();
static void ExportDeviceInfoToScreen();
static void SetStageTimePoint(const std::string &stage_name, const std::string &flag);
static double SetStageTimePointWithReturn(const std::string &stage_name, const std::string &flag);
static void SetStageStatTime(const std::string &stage_name, double cost_time);
static void ExportStageTimePointToFile();
static void ExportStageTimePointToScreen();
static void ExportStageStatTimeToFile();
static void ExportStageStatTimeToScreen();

private:
PynativeProfiler() = default;
~PynativeProfiler() = default;
inline static bool enable_profiler_flag_ = false;
inline static std::vector<std::pair<std::string, std::pair<std::string, double>>> stage_time_point_vec_;
inline static std::vector<std::pair<std::string, double>> stage_stat_time_vec_;
inline static std::vector<std::pair<std::string, std::pair<double, double>>> op_name_launch_time_point_vec_;
inline static std::vector<std::pair<std::string, double>> op_name_launch_time_vec_;
};
} // namespace mindspore

#endif // MINDSPORE_CCSRC_PIPELINE_PYNATIVE_PYNATIVE_PROFILING_H_

+ 4
- 9
mindspore/ccsrc/runtime/device/kernel_runtime.cc View File

@@ -32,7 +32,6 @@
#include "utils/utils.h"
#include "frontend/parallel/context.h"
#include "debug/env_config_parser.h"
#include "pipeline/pynative/pynative_profiling.h"
#if ((defined ENABLE_CPU) && (!defined _WIN32))
#include "ps/ps_cache/ps_cache_manager.h"
#endif
@@ -1309,18 +1308,14 @@ bool KernelRuntime::LaunchKernelWithPynativeProfiling(kernel::KernelMod *kernel_
end->set_record_stream(stream);
start->RecordEvent();
bool ret = kernel_mod->Launch(kernel_launch_info, stream);
if (!ret) {
MS_LOG(EXCEPTION) << "Launch kernel failed, kernel name is : " << op_name;
}
end->RecordEvent();
start->SyncEvent();
end->SyncEvent();
start->ElapsedTime(&cost_time, end.get());
auto launch_end_time = GetTime();
double launch_start_time = launch_end_time - cost_time / kBasicTimeTransferUnit;
auto op_launch_start_time_end_time = std::make_pair(launch_start_time, launch_end_time);
PynativeProfiler::SetDeviceOpNameAndLaunchTimePoint(std::make_pair(op_name, op_launch_start_time_end_time));
PynativeProfiler::SetDeviceOpNameAndLaunchCostTime(std::make_pair(op_name, cost_time / kBasicTimeTransferUnit));
if (!ret) {
MS_LOG(EXCEPTION) << "Launch kernel failed, kernel name is : " << op_name;
}
MS_LOG(DEBUG) << "Launch kernel:" << op_name << " cost:" << cost_time / kBasicTimeTransferUnit;
return ret;
}



+ 2
- 5
mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc View File

@@ -17,7 +17,6 @@
#include "runtime/hardware/gpu/gpu_device_context.h"
#include <dlfcn.h>
#include <utility>
#include "pipeline/pynative/pynative_profiling.h"
#include "runtime/device/gpu/kernel_info_setter.h"
#include "runtime/device/gpu/gpu_kernel_build.h"
#include "runtime/device/gpu/gpu_device_address.h"
@@ -492,10 +491,8 @@ bool GPUDeviceContext::LaunchKernelWithProfiling(const CNodePtr &kernel, const s
profiler_inst->OpDataProducerEnd();

auto op_launch_start_end_time = profiler_inst->GetSingleOpLaunchTime();
std::string op_name = kernel->fullname_with_scope();
PynativeProfiler::SetDeviceOpNameAndLaunchTimePoint(std::make_pair(op_name, op_launch_start_end_time));
PynativeProfiler::SetDeviceOpNameAndLaunchCostTime(
std::make_pair(op_name, op_launch_start_end_time.second - op_launch_start_end_time.first));
MS_LOG(DEBUG) << "Launch kernel:" << kernel->fullname_with_scope() << " cost:"
<< (op_launch_start_end_time.second - op_launch_start_end_time.first) / kBasicTimeTransferUnit;

if (profiler_inst->GetSyncEnableFlag()) {
CHECK_RET_WITH_RETURN_ERROR(SyncStream(), "Profiler SyncStream failed.");


Write Preview
Loading…
Cancel
Save