Browse Source

!7832 Modify profiling file mode to 600

Merge pull request !7832 from yanghaitao/yht_modify_profile_file_mode
tags/v1.1.0
mindspore-ci-bot Gitee 5 years ago
parent
commit
b93264d323
12 changed files with 81 additions and 15 deletions
  1. +9
    -0
      mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_profiling.cc
  2. +1
    -0
      mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_profiling.h
  3. +3
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/connector_size.h
  4. +13
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/connector_throughput.cc
  5. +2
    -15
      mindspore/ccsrc/minddata/dataset/engine/perf/connector_throughput.h
  6. +14
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/dataset_iterator_tracing.cc
  7. +2
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/dataset_iterator_tracing.h
  8. +14
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/device_queue_tracing.cc
  9. +2
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/device_queue_tracing.h
  10. +1
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/monitor.cc
  11. +16
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc
  12. +4
    -0
      mindspore/ccsrc/minddata/dataset/engine/perf/profiling.h

+ 9
- 0
mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_profiling.cc View File

@@ -53,6 +53,15 @@ void GetNextProfiling::SaveProfilingData() {
<< std::endl;
}
handle.close();

ChangeFileMode();
}

void GetNextProfiling::ChangeFileMode() {
if (chmod(common::SafeCStr(file_name_), S_IRUSR | S_IWUSR) == -1) {
MS_LOG(ERROR) << "Modify file:" << file_name_ << " to rw fail.";
return;
}
}

void GetNextProfiling::RecordData(uint32_t queue_size, uint64_t start_time_stamp, uint64_t end_time_stamp) {


+ 1
- 0
mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_profiling.h View File

@@ -35,6 +35,7 @@ class GetNextProfiling : public ProfilingOp {
uint64_t GetTimeStamp() const;
void RecordData(uint32_t queue_size, uint64_t start_time_stamp, uint64_t end_time_stamp);
void Init();
void ChangeFileMode();

private:
std::string profiling_path_;


+ 3
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/connector_size.h View File

@@ -61,6 +61,9 @@ class ConnectorSize : public Sampling {
// Parse op infomation and transform to json format
json ParseOpInfo(const DatasetOp &node, const std::vector<int32_t> &size);

// Change file mode after save throughput data
Status ChangeFileMode() { return Status::OK(); }

private:
ExecutionTree *tree_ = nullptr; // ExecutionTree pointer
ConnectorSizeSampleTable sample_table_; // Dataset structure to store all samples of connector size sampling


+ 13
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/connector_throughput.cc View File

@@ -14,6 +14,7 @@
* limitations under the License.
*/

#include <sys/stat.h>
#include <fstream>
#include <iterator>
#include <algorithm>
@@ -132,5 +133,17 @@ Status ConnectorThroughput::Init(const std::string &dir_path, const std::string
file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + device_id + ".json")).toString();
return Status::OK();
}

Status ConnectorThroughput::ChangeFileMode() {
if (file_path_.empty()) {
return Status::OK();
}

if (chmod(common::SafeCStr(file_path_), S_IRUSR | S_IWUSR) == -1) {
std::string err_str = "Change file mode failed," + file_path_;
return Status(StatusCode::kUnexpectedError, err_str);
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

+ 2
- 15
mindspore/ccsrc/minddata/dataset/engine/perf/connector_throughput.h View File

@@ -59,21 +59,6 @@ class ConnectorThroughput : public Sampling {
// This function samples the connector size of every nodes within the ExecutionTree
Status Sample() override;

/* Status TestPrint() override {
std::ofstream os("performance_monitor.txt");
if (throughput_.size() == 0) {
os << "data is empty" << std::endl;
return Status::OK();
}
for (int i = 0; i < throughput_.size(); i++) {
for (int j = 0; j < n_nodes_; j++) {
os << throughput_[j][i] << " ";
}
os << std::endl;
}
return Status::OK();
};*/

// Traverse the tree nodes and count them
int InitNodes();

@@ -87,6 +72,8 @@ class ConnectorThroughput : public Sampling {

json ParseOpInfo(const DatasetOp &node, const std::vector<double> &thr);

Status ChangeFileMode() override;

private:
ExecutionTree *tree_ = nullptr; // ExecutionTree pointer
int64_t max_rows_;


+ 14
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/dataset_iterator_tracing.cc View File

@@ -13,10 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <sys/stat.h>
#include <fstream>
#include <string>
#include "minddata/dataset/engine/perf/dataset_iterator_tracing.h"
#include "minddata/dataset/util/path.h"
#include "mindspore/core/utils/ms_utils.h"

namespace mindspore {
namespace dataset {
@@ -60,5 +62,17 @@ Status DatasetIteratorTracing::Init(const std::string &dir_path, const std::stri
file_path_ = (Path(dir_path) / Path("dataset_iterator_profiling_" + device_id + ".txt")).toString();
return Status::OK();
}

Status DatasetIteratorTracing::ChangeFileMode() {
if (value_.empty()) {
return Status::OK();
}

if (chmod(common::SafeCStr(file_path_), S_IRUSR | S_IWUSR) == -1) {
std::string err_str = "Change file mode failed," + file_path_;
return Status(StatusCode::kUnexpectedError, err_str);
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

+ 2
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/dataset_iterator_tracing.h View File

@@ -43,6 +43,8 @@ class DatasetIteratorTracing : public Tracing {

Status Init(const std::string &dir_path, const std::string &device_id) override;

Status ChangeFileMode() override;

private:
std::vector<std::string> value_;
};


+ 14
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/device_queue_tracing.cc View File

@@ -14,10 +14,12 @@
* limitations under the License.
*/

#include <sys/stat.h>
#include <fstream>
#include <string>
#include "minddata/dataset/engine/perf/device_queue_tracing.h"
#include "minddata/dataset/util/path.h"
#include "mindspore/core/utils/ms_utils.h"
namespace mindspore {
namespace dataset {

@@ -60,5 +62,17 @@ Status DeviceQueueTracing::Init(const std::string &dir_path, const std::string &
file_path_ = (Path(dir_path) / Path("device_queue_profiling_" + device_id + ".txt")).toString();
return Status::OK();
}

Status DeviceQueueTracing::ChangeFileMode() {
if (value_.empty()) {
return Status::OK();
}

if (chmod(common::SafeCStr(file_path_), S_IRUSR | S_IWUSR) == -1) {
std::string err_str = "Change file mode failed," + file_path_;
return Status(StatusCode::kUnexpectedError, err_str);
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

+ 2
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/device_queue_tracing.h View File

@@ -43,6 +43,8 @@ class DeviceQueueTracing : public Tracing {

Status Init(const std::string &dir_path, const std::string &device_id) override;

Status ChangeFileMode() override;

private:
std::vector<std::string> value_;
};


+ 1
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/monitor.cc View File

@@ -48,6 +48,7 @@ Status Monitor::operator()() {

// Output all profiling data upon request.
tree_->GetProfilingManager()->SaveProfilingData();
tree_->GetProfilingManager()->ChangeFileMode();
return Status::OK();
}



+ 16
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc View File

@@ -14,6 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/engine/perf/profiling.h"
#include <sys/stat.h>
#include <sys/time.h>
#include <cstdlib>
#include <fstream>
@@ -162,6 +163,21 @@ Status ProfilingManager::SaveProfilingData() {
return Status::OK();
}

Status ProfilingManager::ChangeFileMode() {
if (!IsProfilingEnable()) {
return Status::OK();
}
MS_LOG(INFO) << "Start to change file mode.";
for (auto node : tracing_nodes_) {
RETURN_IF_NOT_OK(node.second->ChangeFileMode());
}
for (auto node : sampling_nodes_) {
RETURN_IF_NOT_OK(node.second->ChangeFileMode());
}
MS_LOG(INFO) << "Change file mode end.";
return Status::OK();
}

int64_t ProfilingTime::GetCurMilliSecond() {
// because cpplint does not allow using namespace
using std::chrono::duration_cast;


+ 4
- 0
mindspore/ccsrc/minddata/dataset/engine/perf/profiling.h View File

@@ -52,6 +52,8 @@ class Profiling : std::enable_shared_from_this<Profiling> {
// Profiling name
virtual std::string Name() const = 0;

virtual Status ChangeFileMode() = 0;

protected:
std::string file_path_;
};
@@ -108,6 +110,8 @@ class ProfilingManager {
// Launch monitoring thread.
Status LaunchMonitor();

Status ChangeFileMode();

private:
std::unique_ptr<Monitor> perf_monitor_;
std::unordered_map<std::string, std::shared_ptr<Tracing>> tracing_nodes_;


Loading…
Cancel
Save