Browse Source

Fix cache sharing for more scenarios

tags/v1.0.0
Lixia Chen 5 years ago
parent
commit
ba94ce3fd1
1 changed files with 11 additions and 1 deletions
  1. +11
    -1
      mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc

+ 11
- 1
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc View File

@@ -390,7 +390,8 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {

// Filter out the Num workers field when generating the check sum
ss_str = std::regex_replace(ss_str, std::regex("Num workers.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex("\\[workers.*\\]"), "");
ss_str = std::regex_replace(ss_str, std::regex("\\[workers.*?\\]"), "");
ss_str = std::regex_replace(ss_str, std::regex("Connector queue size.*\n"), "");

// Filter out tcp/ip information
ss_str = std::regex_replace(ss_str, std::regex("Hostname.*\n"), "");
@@ -409,6 +410,15 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {
ss_str = std::regex_replace(ss_str, std::regex("Device id.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex("device_id.*\n"), "");

// Filter out the operator id field
ss_str = std::regex_replace(ss_str, std::regex("Parent.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex("Child.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex(R"(\(\s*\d+?\))"), "");

// Filter out the total repeats and number repeats per epoch field
ss_str = std::regex_replace(ss_str, std::regex("Total repeats.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex("Number repeats per epoch.*\n"), "");

// The Cache crc and Server cache id field is different when creating new cache_client and re-using the same
// cache_client later. So we filter out these two fields to allow cache sharing.
ss_str = std::regex_replace(ss_str, std::regex("Cache crc.*\n"), "");


Loading…
Cancel
Save