Add more ut for transform ops and subset sampler

Signed-off-by: alex-yuyue <yue.yu1@huawei.com>

mindspore/ccsrc/minddata/dataset/include/transforms.h

@@ -159,9 +159,9 @@ class Compose final : public TensorTransform {
 class Concatenate final : public TensorTransform {
  public:
   /// \brief Constructor.
-  /// \param[in] axis Concatenate the tensors along given axis (Default=0).
-  /// \param[in] prepend MSTensor to be prepended to the already concatenated tensors (Default={}).
-  /// \param[in] append MSTensor to be appended to the already concatenated tensors (Default={}).
+  /// \param[in] axis Concatenate the tensors along given axis, only support 0 or -1 so far (default=0).
+  /// \param[in] prepend MSTensor to be prepended to the already concatenated tensors (default={}).
+  /// \param[in] append MSTensor to be appended to the already concatenated tensors (default={}).
   explicit Concatenate(int8_t axis = 0, MSTensor prepend = {}, MSTensor append = {});
 
   /// \brief Destructor
@@ -227,7 +227,8 @@ class Mask final : public TensorTransform {
   /// \param[in] op One of the relational operators EQ, NE LT, GT, LE or GE.
   /// \param[in] constant Constant to be compared to.
   ///               Can only be MSTensor of str, int, float, bool.
-  /// \param[in] de_type Type of the generated mask (Default to be mindspore::DataType::kNumberTypeBool).
+  /// \param[in] de_type Type of the generated mask. Can only be numeric or boolean datatype.
+  ///               (default=mindspore::DataType::kNumberTypeBool)
   explicit Mask(RelationalOp op, MSTensor constant,
                 mindspore::DataType ms_type = mindspore::DataType(mindspore::DataType::kNumberTypeBool));
 
@@ -273,7 +274,7 @@ class PadEnd final : public TensorTransform {
   /// \param[in] pad_shape List of integers representing the shape needed.
   ///               Dimensions that set to `None` will not be padded (i.e., original dim will be used).
   ///               Shorter dimensions will truncate the values.
-  /// \param[in] pad_value Value used to pad. Default to be {}.
+  /// \param[in] pad_value Value used to pad (default={}).
   explicit PadEnd(const std::vector<dsize_t> &pad_shape, MSTensor pad_value = {});
 
   /// \brief Destructor

mindspore/ccsrc/minddata/dataset/kernels/ir/data/transforms_ir.cc

@@ -139,7 +139,14 @@ Status FillOperation::to_json(nlohmann::json *out_json) {
 MaskOperation::MaskOperation(RelationalOp op, const std::shared_ptr<Tensor> &constant, DataType dtype)
     : op_(op), constant_(constant), dtype_(dtype) {}
 
-Status MaskOperation::ValidateParams() { return Status::OK(); }
+Status MaskOperation::ValidateParams() {
+  if (!dtype_.IsBool() && !dtype_.IsFloat() && !dtype_.IsInt()) {
+    std::string err_msg = "Mask: Only supports bool or numeric datatype for generated mask type.";
+    MS_LOG(ERROR) << err_msg;
+    RETURN_STATUS_SYNTAX_ERROR(err_msg);
+  }
+  return Status::OK();
+}
 
 std::shared_ptr<TensorOp> MaskOperation::Build() { return std::make_shared<MaskOp>(op_, constant_, dtype_); }
 #endif

mindspore/dataset/transforms/c_transforms.py

@@ -163,7 +163,7 @@ class _SliceOption(cde.SliceOption):
         super().__init__(slice_option)
 
 
-class Slice():
+class Slice(TensorOperation):
     """
     Slice operation to extract a tensor out using the given n slices.
 
@@ -226,7 +226,7 @@ DE_C_RELATIONAL = {Relational.EQ: cde.RelationalOp.EQ,
                    Relational.LE: cde.RelationalOp.LE}
 
 
-class Mask():
+class Mask(TensorOperation):
     """
     Mask content of the input tensor with the given predicate.
     Any element of the tensor that matches the predicate will be evaluated to True, otherwise False.
@@ -264,7 +264,7 @@ class Mask():
         return cde.MaskOperation(DE_C_RELATIONAL[self.operator], self.constant, self.dtype)
 
 
-class PadEnd():
+class PadEnd(TensorOperation):
     """
     Pad input tensor according to pad_shape, need to have same rank.
 
@@ -300,7 +300,7 @@ class PadEnd():
         return cde.PadEndOperation(self.pad_shape, self.pad_value)
 
 
-class Concatenate():
+class Concatenate(TensorOperation):
     """
     Tensor operation that concatenates all columns into a single tensor.
 

tests/ut/cpp/dataset/c_api_samplers_test.cc

@@ -325,3 +325,119 @@ TEST_F(MindDataTestPipeline, TestSamplerAddChild) {
   EXPECT_EQ(ds->GetDatasetSize(), 5);
   iter->Stop();
 }
+
+TEST_F(MindDataTestPipeline, TestSubsetSamplerSuccess1) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSubsetSamplerSuccess1.";
+  // Test basic setting of subset_sampler with default num_samples
+
+  std::vector<int64_t> indices = {2, 4, 6, 8, 10, 12};
+  std::shared_ptr<Sampler> sampl = std::make_shared<SubsetSampler>(indices);
+  EXPECT_NE(sampl, nullptr);
+
+  // Create an ImageFolder Dataset
+  std::string folder_path = datasets_root_path_ + "/testPK/data/";
+  std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampl);
+  EXPECT_NE(ds, nullptr);
+
+  // Iterate the dataset and get each row
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  EXPECT_NE(iter, nullptr);
+  std::unordered_map<std::string, mindspore::MSTensor> row;
+  ASSERT_OK(iter->GetNextRow(&row));
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
+    ASSERT_OK(iter->GetNextRow(&row));
+  }
+
+  EXPECT_EQ(i, 6);
+  iter->Stop();
+}
+
+TEST_F(MindDataTestPipeline, TestSubsetSamplerSuccess2) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSubsetSamplerSuccess2.";
+  // Test subset_sampler with num_samples
+
+  std::vector<int64_t> indices = {2, 4, 6, 8, 10, 12};
+  std::shared_ptr<Sampler> sampl = std::make_shared<SubsetSampler>(indices, 3);
+  EXPECT_NE(sampl, nullptr);
+
+  // Create an ImageFolder Dataset
+  std::string folder_path = datasets_root_path_ + "/testPK/data/";
+  std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampl);
+  EXPECT_NE(ds, nullptr);
+
+  // Iterate the dataset and get each row
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  EXPECT_NE(iter, nullptr);
+  std::unordered_map<std::string, mindspore::MSTensor> row;
+  ASSERT_OK(iter->GetNextRow(&row));
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
+    ASSERT_OK(iter->GetNextRow(&row));
+  }
+
+  EXPECT_EQ(i, 3);
+  iter->Stop();
+}
+
+TEST_F(MindDataTestPipeline, TestSubsetSamplerSuccess3) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSubsetSamplerSuccess3.";
+  // Test subset_sampler with num_samples larger than the indices size.
+
+  std::vector<int64_t> indices = {2, 4, 6, 8, 10, 12};
+  std::shared_ptr<Sampler> sampl = std::make_shared<SubsetSampler>(indices, 8);
+  EXPECT_NE(sampl, nullptr);
+
+  // Create an ImageFolder Dataset
+  std::string folder_path = datasets_root_path_ + "/testPK/data/";
+  std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampl);
+  EXPECT_NE(ds, nullptr);
+
+  // Iterate the dataset and get each row
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  EXPECT_NE(iter, nullptr);
+  std::unordered_map<std::string, mindspore::MSTensor> row;
+  ASSERT_OK(iter->GetNextRow(&row));
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
+    ASSERT_OK(iter->GetNextRow(&row));
+  }
+
+  EXPECT_EQ(i, 6);
+  iter->Stop();
+}
+
+TEST_F(MindDataTestPipeline, TestSubsetSamplerFail) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSubsetSamplerFail.";
+  // Test subset_sampler with index out of bounds.
+
+  std::vector<int64_t> indices = {2, 4, 6, 8, 10, 100};  // Sample ID (100) is out of bound
+  std::shared_ptr<Sampler> sampl = std::make_shared<SubsetSampler>(indices);
+  EXPECT_NE(sampl, nullptr);
+
+  // Create an ImageFolder Dataset
+  std::string folder_path = datasets_root_path_ + "/testPK/data/";
+  std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampl);
+  EXPECT_NE(ds, nullptr);
+
+  // Iterate the dataset and get each row
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  EXPECT_NE(iter, nullptr);
+  std::unordered_map<std::string, mindspore::MSTensor> row;
+  // Expect failure: index 100 is out of dataset bounds
+  EXPECT_ERROR(iter->GetNextRow(&row));
+
+  iter->Stop();
+}

tests/ut/python/dataset/test_mask_op.py

@@ -121,7 +121,7 @@ def test_mask_exceptions_str():
 
     with pytest.raises(RuntimeError) as info:
         mask_compare(["1", "2", "3", "4", "5"], ops.Relational.EQ, "3.5", mstype.string)
-    assert "only support numeric datatype of input." in str(info.value)
+    assert "Only supports bool or numeric datatype for generated mask type." in str(info.value)
 
 
 if __name__ == "__main__":