Same runtime CacheClient if c++ cache is shared

mindspore/ccsrc/minddata/dataset/api/python/pybind_conversion.cc

@@ -147,7 +147,6 @@ std::shared_ptr<SamplerObj> toSamplerObj(std::optional<py::handle> py_sampler, b
 std::shared_ptr<DatasetCache> toDatasetCache(std::optional<std::shared_ptr<CacheClient>> cc) {
   if (cc) {
     std::shared_ptr<DatasetCache> built_cache;
-    // Common Sampler
     built_cache = std::make_shared<PreBuiltDatasetCache>(std::move(cc.value()));
     return built_cache;
   } else {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc

@@ -558,7 +558,8 @@ void AlbumOp::Print(std::ostream &out, bool show_all) const {
     // Call the super class for displaying any common detailed info
     ParallelOp::Print(out, show_all);
     // Then show any custom derived-internal stuff
-    out << "\nNumber of rows:" << num_rows_ << "\nAlbum directory: " << folder_path_ << "\n\n";
+    out << "\nNumber of rows:" << num_rows_ << "\nAlbum directory: " << folder_path_
+        << "\nDecode: " << (decode_ ? "yes" : "no") << "\n\n";
   }
 }
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc

@@ -422,7 +422,8 @@ void CelebAOp::Print(std::ostream &out, bool show_all) const {
     // Call the super class for displaying any common detailed info
     ParallelOp::Print(out, show_all);
     // Then show any custom derived-internal stuff
-    out << "\nNumber of rows:" << num_rows_ << "\nceleba dir: " << folder_path_ << "\n\n";
+    out << "\nNumber of rows:" << num_rows_ << "\nceleba dir: " << folder_path_
+        << "\nDecode: " << (decode_ ? "yes" : "no") << "\n\n";
   }
 }
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc

@@ -210,7 +210,8 @@ void CocoOp::Print(std::ostream &out, bool show_all) const {
     // Call the super class for displaying any common detailed info
     ParallelOp::Print(out, show_all);
     // Then show any custom derived-internal stuff
-    out << "\nNumber of rows: " << num_rows_ << "\nCOCO Directory: " << image_folder_path_ << "\n\n";
+    out << "\nNumber of rows: " << num_rows_ << "\nCOCO Directory: " << image_folder_path_
+        << "\nDecode: " << (decode_ ? "yes" : "no") << "\n\n";
   }
 }
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc

@@ -255,7 +255,8 @@ void ImageFolderOp::Print(std::ostream &out, bool show_all) const {
     // Call the super class for displaying any common detailed info
     ParallelOp::Print(out, show_all);
     // Then show any custom derived-internal stuff
-    out << "\nNumber of rows:" << num_rows_ << "\nImageFolder directory: " << folder_path_ << "\n\n";
+    out << "\nNumber of rows:" << num_rows_ << "\nImageFolder directory: " << folder_path_
+        << "\nDecode: " << (decode_ ? "yes" : "no") << "\n\n";
   }
 }
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc

@@ -244,7 +244,8 @@ void ManifestOp::Print(std::ostream &out, bool show_all) const {
     // Call the super class for displaying any common detailed info
     ParallelOp::Print(out, show_all);
     // Then show any custom derived-internal stuff
-    out << "\nNumber of rows:" << num_rows_ << "\nManifest file: " << file_ << "\n\n";
+    out << "\nNumber of rows:" << num_rows_ << "\nManifest file: " << file_ << "\nDecode: " << (decode_ ? "yes" : "no")
+        << "\n\n";
   }
 }
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc

@@ -191,7 +191,8 @@ void VOCOp::Print(std::ostream &out, bool show_all) const {
     // Call the super class for displaying any common detailed info
     ParallelOp::Print(out, show_all);
     // Then show any custom derived-internal stuff
-    out << "\nNumber of rows: " << num_rows_ << "\nVOC Directory: " << folder_path_ << "\n\n";
+    out << "\nNumber of rows: " << num_rows_ << "\nVOC Directory: " << folder_path_
+        << "\nDecode: " << (decode_ ? "yes" : "no") << "\n\n";
   }
 }
 

mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.cc

@@ -23,6 +23,10 @@ namespace dataset {
 /// Method to initialize the DatasetCache by creating an instance of a CacheClient
 /// \return Status Error code
 Status DatasetCacheImpl::Build() {
+  // The same DatasetCache instance can be re-used for multiple pipelines for cache sharing,
+  // in this case, cache_client_ object might have been created.
+  if (cache_client_) return Status::OK();
+
   CacheClient::Builder builder;
   builder.SetSessionId(session_id_).SetCacheMemSz(cache_mem_sz_).SetSpill(spill_);
   if (hostname_) builder.SetHostname(hostname_.value());

tests/ut/cpp/dataset/c_api_cache_test.cc

@@ -735,3 +735,129 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheClueCApi) {
   // Manually terminate the pipeline
   iter->Stop();
 }
+
+TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare) {
+  session_id_type env_session;
+  Status s = GetSessionFromEnv(&env_session);
+  EXPECT_EQ(s, Status::OK());
+
+  std::shared_ptr<DatasetCache> some_cache = CreateDatasetCache(env_session, 0, true);
+  EXPECT_NE(some_cache, nullptr);
+
+  // Create an ImageFolder Dataset, this folder_path only has 2 images in it
+  std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
+  std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
+  EXPECT_NE(ds1, nullptr);
+  std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
+  EXPECT_NE(ds2, nullptr);
+
+  // Create and launch the Execution Tree for ds1
+  std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
+  EXPECT_NE(iter1, nullptr);
+  // Iterate the dataset and get each row
+  std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
+  iter1->GetNextRow(&row);
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image shape: " << image->shape();
+    iter1->GetNextRow(&row);
+  }
+  EXPECT_EQ(i, 2);
+  // Manually terminate the pipeline
+  iter1->Stop();
+
+  // Create and launch the Execution Tree for ds2
+  std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
+  EXPECT_NE(iter2, nullptr);
+  // Iterate the dataset and get each row
+  iter2->GetNextRow(&row);
+
+  i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image shape: " << image->shape();
+    iter2->GetNextRow(&row);
+  }
+  EXPECT_EQ(i, 2);
+
+  // Manually terminate the pipeline
+  iter2->Stop();
+}
+
+TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) {
+  session_id_type env_session;
+  Status s = GetSessionFromEnv(&env_session);
+  EXPECT_EQ(s, Status::OK());
+
+  std::shared_ptr<DatasetCache> some_cache = CreateDatasetCache(env_session, 0, true);
+  EXPECT_NE(some_cache, nullptr);
+
+  // Create an ImageFolder Dataset, this folder_path only has 2 images in it
+  std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
+  std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache);
+  EXPECT_NE(ds1, nullptr);
+  std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, SequentialSampler(), {}, {}, some_cache);
+  EXPECT_NE(ds2, nullptr);
+
+  // Create and launch the Execution Tree for ds1
+  std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
+  EXPECT_NE(iter1, nullptr);
+  // Iterate the dataset and get each row
+  std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
+  iter1->GetNextRow(&row);
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    iter1->GetNextRow(&row);
+  }
+  EXPECT_EQ(i, 2);
+  // Manually terminate the pipeline
+  iter1->Stop();
+
+  // Re-use a cache for the second pipeline would fail
+  std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
+  EXPECT_EQ(iter2, nullptr);
+}
+
+TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure2) {
+  session_id_type env_session;
+  Status s = GetSessionFromEnv(&env_session);
+  EXPECT_EQ(s, Status::OK());
+
+  std::shared_ptr<DatasetCache> some_cache = CreateDatasetCache(env_session, 0, true);
+  EXPECT_NE(some_cache, nullptr);
+
+  // Create an ImageFolder Dataset, this folder_path only has 2 images in it
+  std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
+  std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache);
+  EXPECT_NE(ds1, nullptr);
+  std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
+  EXPECT_NE(ds2, nullptr);
+
+  // Create and launch the Execution Tree for ds1
+  std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
+  EXPECT_NE(iter1, nullptr);
+  // Iterate the dataset and get each row
+  std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
+  iter1->GetNextRow(&row);
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    iter1->GetNextRow(&row);
+  }
+  EXPECT_EQ(i, 2);
+  // Manually terminate the pipeline
+  iter1->Stop();
+
+  // Re-use a cache for the second pipeline would fail
+  std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
+  EXPECT_EQ(iter2, nullptr);
+}