Support 3 composed data ops and cleanup code

mindspore/ccsrc/minddata/dataset/api/datasets.cc

@@ -483,7 +483,6 @@ FilterDataset::FilterDataset(std::shared_ptr<Dataset> input, std::function<Tenso
 }
 #endif
 
-// FIXME - Should be removed once all Tensor op API class has been added
 MapDataset::MapDataset(std::shared_ptr<Dataset> input, std::vector<std::shared_ptr<TensorOperation>> operations,
                        const std::vector<std::string> &input_columns, const std::vector<std::string> &output_columns,
                        const std::vector<std::string> &project_columns, const std::shared_ptr<DatasetCache> &cache,

mindspore/ccsrc/minddata/dataset/api/execute.cc

@@ -21,6 +21,7 @@
 #include "minddata/dataset/core/tensor_row.h"
 #include "minddata/dataset/include/tensor.h"
 #include "minddata/dataset/include/type_id.h"
+#include "minddata/dataset/kernels/ir/tensor_operation.h"
 #include "minddata/dataset/kernels/tensor_op.h"
 #ifndef ENABLE_ANDROID
 #include "utils/log_adapter.h"

mindspore/ccsrc/minddata/dataset/api/transforms.cc

@@ -16,22 +16,40 @@
 
 #include "minddata/dataset/include/transforms.h"
 
+#include <algorithm>
+
+#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
+
 namespace mindspore {
 namespace dataset {
 
 // Transform operations for data.
 namespace transforms {
 
-// FUNCTIONS TO CREATE DATA TRANSFORM OPERATIONS
+// API CLASS FOR DATA TRANSFORM OPERATIONS
 // (In alphabetical order)
 
-// Function to create ComposeOperation.
-std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
-  auto op = std::make_shared<ComposeOperation>(transforms);
-  // Input validation
-  return op->ValidateParams() ? op : nullptr;
+// Constructor to Compose.
+Compose::Compose(const std::vector<TensorTransform *> &transforms) {
+  (void)std::transform(
+    transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+    [](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
+}
+
+Compose::Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms) {
+  (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+                       [](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
+                         return op != nullptr ? op->Parse() : nullptr;
+                       });
+}
+
+Compose::Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms) {
+  (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+                       [](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
 }
 
+std::shared_ptr<TensorOperation> Compose::Parse() { return std::make_shared<ComposeOperation>(transforms_); }
+
 // Constructor to Duplicate
 Duplicate::Duplicate() {}
 
@@ -42,31 +60,66 @@ OneHot::OneHot(int32_t num_classes) : num_classes_(num_classes) {}
 
 std::shared_ptr<TensorOperation> OneHot::Parse() { return std::make_shared<OneHotOperation>(num_classes_); }
 
-// Function to create RandomApplyOperation.
-std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
-                                                  double prob) {
-  auto op = std::make_shared<RandomApplyOperation>(transforms, prob);
-  // Input validation
-  return op->ValidateParams() ? op : nullptr;
+// Constructor to RandomApply.
+RandomApply::RandomApply(const std::vector<TensorTransform *> &transforms, double prob) : prob_(prob) {
+  (void)std::transform(
+    transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+    [](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
+}
+
+RandomApply::RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob) : prob_(prob) {
+  (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+                       [](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
+                         return op != nullptr ? op->Parse() : nullptr;
+                       });
+}
+
+RandomApply::RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob)
+    : prob_(prob) {
+  (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+                       [](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
 }
 
-// Function to create RandomChoiceOperation.
-std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
-  auto op = std::make_shared<RandomChoiceOperation>(transforms);
-  // Input validation
-  return op->ValidateParams() ? op : nullptr;
+std::shared_ptr<TensorOperation> RandomApply::Parse() {
+  return std::make_shared<RandomApplyOperation>(transforms_, prob_);
 }
 
+// Constructor to RandomChoice.
+RandomChoice::RandomChoice(const std::vector<TensorTransform *> &transforms) {
+  (void)std::transform(
+    transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+    [](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
+}
+
+RandomChoice::RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms) {
+  (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+                       [](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
+                         return op != nullptr ? op->Parse() : nullptr;
+                       });
+}
+
+RandomChoice::RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms) {
+  (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_),
+                       [](TensorTransform &op) -> std::shared_ptr<TensorOperation> { return op.Parse(); });
+}
+
+std::shared_ptr<TensorOperation> RandomChoice::Parse() { return std::make_shared<RandomChoiceOperation>(transforms_); }
+
 // Constructor to TypeCast
 TypeCast::TypeCast(std::string data_type) : data_type_(data_type) {}
 
 std::shared_ptr<TensorOperation> TypeCast::Parse() { return std::make_shared<TypeCastOperation>(data_type_); }
 
-#ifndef ENABLE_ANDROID
 // Constructor to Unique
 Unique::Unique() {}
 
+#ifndef ENABLE_ANDROID
 std::shared_ptr<TensorOperation> Unique::Parse() { return std::make_shared<UniqueOperation>(); }
+#else
+std::shared_ptr<TensorOperation> Unique::Parse() {
+  MS_LOG(ERROR) << "Unique op is not supported for Android.";
+  return nullptr;
+}
 #endif
 }  // namespace transforms
 }  // namespace dataset

mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc

@@ -24,7 +24,7 @@
 
 #include "minddata/dataset/engine/datasetops/map_op/map_op.h"
 #include "minddata/dataset/engine/opt/pass.h"
-#include "minddata/dataset/include/transforms.h"
+#include "minddata/dataset/kernels/ir/tensor_operation.h"
 #include "minddata/dataset/util/status.h"
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc

@@ -26,6 +26,7 @@
 #include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
 #include "minddata/dataset/engine/ir/datasetops/take_node.h"
 #include "minddata/dataset/engine/ir/datasetops/zip_node.h"
+#include "minddata/dataset/kernels/ir/tensor_operation.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/include/datasets.h

@@ -295,14 +295,18 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
   /// \param[in] project_columns A list of column names to project
   /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
   /// \return Shared pointer to the current MapDataset
-  std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorOperation>> operations,
+  std::shared_ptr<MapDataset> Map(std::vector<TensorTransform *> operations,
                                   const std::vector<std::string> &input_columns = {},
                                   const std::vector<std::string> &output_columns = {},
                                   const std::vector<std::string> &project_columns = {},
                                   const std::shared_ptr<DatasetCache> &cache = nullptr,
                                   std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
-    return std::make_shared<MapDataset>(shared_from_this(), operations, input_columns, output_columns, project_columns,
-                                        cache, callbacks);
+    std::vector<std::shared_ptr<TensorOperation>> transform_ops;
+    (void)std::transform(
+      operations.begin(), operations.end(), std::back_inserter(transform_ops),
+      [](TensorTransform *op) -> std::shared_ptr<TensorOperation> { return op != nullptr ? op->Parse() : nullptr; });
+    return std::make_shared<MapDataset>(shared_from_this(), transform_ops, input_columns, output_columns,
+                                        project_columns, cache, callbacks);
   }
 
   std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorTransform>> operations,
@@ -312,9 +316,10 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
                                   const std::shared_ptr<DatasetCache> &cache = nullptr,
                                   std::vector<std::shared_ptr<DSCallback>> callbacks = {}) {
     std::vector<std::shared_ptr<TensorOperation>> transform_ops;
-    (void)std::transform(
-      operations.begin(), operations.end(), std::back_inserter(transform_ops),
-      [](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> { return op->Parse(); });
+    (void)std::transform(operations.begin(), operations.end(), std::back_inserter(transform_ops),
+                         [](std::shared_ptr<TensorTransform> op) -> std::shared_ptr<TensorOperation> {
+                           return op != nullptr ? op->Parse() : nullptr;
+                         });
     return std::make_shared<MapDataset>(shared_from_this(), transform_ops, input_columns, output_columns,
                                         project_columns, cache, callbacks);
   }

mindspore/ccsrc/minddata/dataset/include/text.h

@@ -60,7 +60,7 @@ class BasicTokenizer : public TensorTransform {
   ~BasicTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -100,7 +100,7 @@ class BertTokenizer : public TensorTransform {
   ~BertTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -126,7 +126,7 @@ class CaseFold : public TensorTransform {
   ~CaseFold() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  //// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 };
 #endif
@@ -152,7 +152,7 @@ class JiebaTokenizer : public TensorTransform {
   ~JiebaTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
   Status AddWord(const std::string &word, int64_t freq = 0);
@@ -181,7 +181,7 @@ class Lookup : public TensorTransform {
   ~Lookup() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -209,7 +209,7 @@ class Ngram : public TensorTransform {
   ~Ngram() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -239,7 +239,7 @@ class NormalizeUTF8 : public TensorTransform {
   ~NormalizeUTF8() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -260,7 +260,7 @@ class RegexReplace : public TensorTransform {
   ~RegexReplace() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -284,7 +284,7 @@ class RegexTokenizer : public TensorTransform {
   ~RegexTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -312,7 +312,7 @@ class SentencePieceTokenizer : public TensorTransform {
   ~SentencePieceTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -336,7 +336,7 @@ class SlidingWindow : public TensorTransform {
   ~SlidingWindow() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -359,7 +359,7 @@ class ToNumber : public TensorTransform {
   ~ToNumber() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -377,7 +377,7 @@ class TruncateSequencePair : public TensorTransform {
   ~TruncateSequencePair() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -395,7 +395,7 @@ class UnicodeCharTokenizer : public TensorTransform {
   ~UnicodeCharTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -415,7 +415,7 @@ class UnicodeScriptTokenizer : public TensorTransform {
   ~UnicodeScriptTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
@@ -434,7 +434,7 @@ class WhitespaceTokenizer : public TensorTransform {
   ~WhitespaceTokenizer() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:

mindspore/ccsrc/minddata/dataset/include/transforms.h

@@ -25,11 +25,11 @@
 #include "include/api/status.h"
 #include "minddata/dataset/include/constants.h"
 
-// FIXME - This internal IR header will be removed when external API classes are provided
-#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
-
 namespace mindspore {
 namespace dataset {
+
+class TensorOperation;
+
 // Abstract class to represent a tensor transform operation in the data pipeline.
 /// \class TensorTransform transforms.h
 /// \brief A base class to represent a tensor transform operation in the data pipeline.
@@ -54,16 +54,26 @@ class TensorTransform : public std::enable_shared_from_this<TensorTransform> {
 // Transform operations for performing data transformation.
 namespace transforms {
 
-// Transform Op classes (in alphabetical order)
-class ComposeOperation;
-class RandomApplyOperation;
-class RandomChoiceOperation;
-
-/// \brief Function to create a Compose TensorOperation.
+/// \brief Compose Op.
 /// \notes Compose a list of transforms into a single transform.
-/// \param[in] transforms A vector of transformations to be applied.
-/// \return Shared pointer to the current TensorOperation.
-std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
+class Compose : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] transforms A vector of transformations to be applied.
+  explicit Compose(const std::vector<TensorTransform *> &transforms);
+  explicit Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
+  explicit Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
+
+  /// \brief Destructor
+  ~Compose() = default;
+
+  /// \brief Function to convert TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  std::vector<std::shared_ptr<TensorOperation>> transforms_;
+};
 
 /// \brief Duplicate Op.
 /// \notes Duplicate the input tensor to a new output tensor.
@@ -77,7 +87,7 @@ class Duplicate : public TensorTransform {
   ~Duplicate() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 };
 
@@ -93,26 +103,56 @@ class OneHot : public TensorTransform {
   ~OneHot() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
   float num_classes_;
 };
 
-/// \brief Function to create a RandomApply TensorOperation.
+/// \brief RandomApply Op.
 /// \notes Randomly perform a series of transforms with a given probability.
-/// \param[in] transforms A vector of transformations to be applied.
-/// \param[in] prob The probability to apply the transformation list (default=0.5)
-/// \return Shared pointer to the current TensorOperation.
-std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
-                                                  double prob = 0.5);
+class RandomApply : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] transforms A vector of transformations to be applied.
+  /// \param[in] prob The probability to apply the transformation list (default=0.5)
+  explicit RandomApply(const std::vector<TensorTransform *> &transforms, double prob = 0.5);
+  explicit RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob = 0.5);
+  explicit RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob = 0.5);
+
+  /// \brief Destructor
+  ~RandomApply() = default;
 
-/// \brief Function to create a RandomChoice TensorOperation.
+  /// \brief Function to convert TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  std::vector<std::shared_ptr<TensorOperation>> transforms_;
+  double prob_;
+};
+
+/// \brief RandomChoice Op.
 /// \notes Randomly selects one transform from a list of transforms to perform operation.
-/// \param[in] transforms A vector of transformations to be chosen from to apply.
-/// \return Shared pointer to the current TensorOperation.
-std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
+class RandomChoice : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] transforms A vector of transformations to be chosen from to apply.
+  explicit RandomChoice(const std::vector<TensorTransform *> &transforms);
+  explicit RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
+  explicit RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
+
+  /// \brief Destructor
+  ~RandomChoice() = default;
+
+  /// \brief Function to convert TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  std::vector<std::shared_ptr<TensorOperation>> transforms_;
+};
 
 /// \brief TypeCast Op.
 /// \notes Tensor operation to cast to a given MindSpore data type.
@@ -126,14 +166,13 @@ class TypeCast : public TensorTransform {
   ~TypeCast() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 
  private:
   std::string data_type_;
 };
 
-#ifndef ENABLE_ANDROID
 /// \brief Unique Op.
 /// \notes Return an output tensor containing all the unique elements of the input tensor in
 ///     the same order that they occur in the input tensor.
@@ -146,10 +185,9 @@ class Unique : public TensorTransform {
   ~Unique() = default;
 
   /// \brief Function to convert TensorTransform object into a TensorOperation object.
-  /// \return return code
+  /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 };
-#endif
 }  // namespace transforms
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc

@@ -184,6 +184,8 @@ Status ValidateVectorTransforms(const std::string &op_name,
         op_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr.";
       MS_LOG(ERROR) << err_msg;
       RETURN_STATUS_SYNTAX_ERROR(err_msg);
+    } else {
+      RETURN_IF_NOT_OK(transforms[i]->ValidateParams());
     }
   }
 

mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h

@@ -22,13 +22,12 @@
 #include <vector>
 
 #include "minddata/dataset/core/tensor.h"
+#include "minddata/dataset/kernels/ir/tensor_operation.h"
 #include "minddata/dataset/util/status.h"
 
 namespace mindspore {
 namespace dataset {
 
-class TensorOperation;
-
 // Helper function to validate probability
 Status ValidateProbability(const std::string &op_name, const float probability);
 

tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc

@@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
   //                                    TensorShape({211653}), TensorShape({1, 4})};
 
   uint64_t i = 0;
-  uint64_t j = 0;
+  // uint64_t j = 0;
   while (row.size() != 0) {
     // MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
     // EXPECT_EQ(2, row.size());

tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc

@@ -238,7 +238,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
   iter->GetNextRow(&row);
 
   uint64_t i = 0;
-  int32_t label_idx = 0;
+  // int32_t label_idx = 0;
   while (row.size() != 0) {
     i++;
     // auto image = row["image"];

tests/ut/cpp/dataset/c_api_transforms_test.cc

@@ -35,10 +35,11 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {
   std::string folder_path = datasets_root_path_ + "/testPK/data/";
   std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 3));
   EXPECT_NE(ds, nullptr);
-  /* FIXME - Disable until proper external API for Compose is provided
+
   // Create objects for the tensor ops
-  std::shared_ptr<TensorOperation> compose = transforms::Compose({vision::Decode(), vision::Resize({777, 777})});
-  EXPECT_NE(compose, nullptr);
+  auto decode_op(new vision::Decode());
+  auto resize_op(new vision::Resize({777, 777}));
+  transforms::Compose compose({decode_op, resize_op});
 
   // Create a Map operation on ds
   ds = ds->Map({compose}, {"image"});
@@ -69,25 +70,71 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {
 
   // Manually terminate the pipeline
   iter->Stop();
-  */
 }
 
-TEST_F(MindDataTestPipeline, TestComposeFail) {
-  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail with invalid transform.";
-  /* FIXME - Disable until proper external API for Compose is provided
+TEST_F(MindDataTestPipeline, TestComposeFail1) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail1 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
+
   // Resize: Non-positive size value: -1 at element: 0
   // Compose: transform ops must not be null
-  std::shared_ptr<TensorOperation> compose1 = transforms::Compose({vision::Decode(), vision::Resize({-1})});
-  EXPECT_EQ(compose1, nullptr);
+  auto decode_op = vision::Decode();
+  auto resize_op = vision::Resize({-1});
+  auto compose = transforms::Compose({decode_op, resize_op});
+
+  // Create a Map operation on ds
+  ds = ds->Map({compose}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid Compose parameter(invalid transform op)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestComposeFail2) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail2 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
 
   // Compose: transform ops must not be null
-  std::shared_ptr<TensorOperation> compose2 = transforms::Compose({vision::Decode(), nullptr});
-  EXPECT_EQ(compose2, nullptr);
+  std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
+  std::shared_ptr<TensorTransform> compose(new transforms::Compose({decode_op, nullptr}));
+
+  // Create a Map operation on ds
+  ds = ds->Map({compose}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid Compose parameter (transform ops must not be null)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestComposeFail3) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail3 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
 
   // Compose: transform list must not be empty
-  std::shared_ptr<TensorOperation> compose3 = transforms::Compose({});
-  EXPECT_EQ(compose3, nullptr);
-  */
+  std::vector<std::shared_ptr<TensorTransform>> list = {};
+  auto compose = transforms::Compose(list);
+
+  // Create a Map operation on ds
+  ds = ds->Map({compose}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid Compose parameter (transform list must not be empty)
+  EXPECT_EQ(iter, nullptr);
 }
 
 TEST_F(MindDataTestPipeline, TestDuplicateSuccess) {
@@ -296,10 +343,10 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {
   std::string folder_path = datasets_root_path_ + "/testPK/data/";
   std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
   EXPECT_NE(ds, nullptr);
-  /* FIXME - Disable until proper external API for RandomApply is provided
+
   // Create objects for the tensor ops
-  std::shared_ptr<TensorOperation> random_apply = transforms::RandomApply({vision::Resize({777, 777})}, 0.8);
-  EXPECT_NE(random_apply, nullptr);
+  auto resize_op = vision::Resize({777, 777});
+  auto random_apply = transforms::RandomApply({resize_op}, 0.8);
 
   // Create a Map operation on ds
   ds = ds->Map({random_apply}, {"image"});
@@ -328,29 +375,92 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {
 
   // Manually terminate the pipeline
   iter->Stop();
-  */
 }
 
-TEST_F(MindDataTestPipeline, TestRandomApplyFail) {
-  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail with invalid transform.";
-  /* FIXME - Disable until proper external API for RandomApply is provided
+TEST_F(MindDataTestPipeline, TestRandomApplyFail1) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail1 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
+
   // Resize: Non-positive size value: -1 at element: 0
   // RandomApply: transform ops must not be null
-  std::shared_ptr<TensorOperation> random_apply1 = transforms::RandomApply({vision::Decode(), vision::Resize({-1})});
-  EXPECT_EQ(random_apply1, nullptr);
+  auto decode_op = vision::Decode();
+  auto resize_op = vision::Resize({-1});
+  auto random_apply = transforms::RandomApply({decode_op, resize_op});
+
+  // Create a Map operation on ds
+  ds = ds->Map({random_apply}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (transform ops must not be null)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestRandomApplyFail2) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail2 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
 
   // RandomApply: transform ops must not be null
-  std::shared_ptr<TensorOperation> random_apply2 = transforms::RandomApply({vision::Decode(), nullptr});
-  EXPECT_EQ(random_apply2, nullptr);
+  std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
+  std::shared_ptr<TensorTransform> random_apply(new transforms::RandomApply({decode_op, nullptr}));
 
-  // RandomApply: transform list must not be empty
-  std::shared_ptr<TensorOperation> random_apply3 = transforms::RandomApply({});
-  EXPECT_EQ(random_apply3, nullptr);
+  // Create a Map operation on ds
+  ds = ds->Map({random_apply}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (transform ops must not be null)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestRandomApplyFail3) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail3 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
 
   // RandomApply: Probability has to be between 0 and 1
-  std::shared_ptr<TensorOperation> random_apply4 = transforms::RandomApply({vision::Resize({100})}, -1);
-  EXPECT_EQ(random_apply4, nullptr);
-  */
+  auto resize_op = vision::Resize({100});
+  auto random_apply = transforms::RandomApply({resize_op}, -1);
+
+  // Create a Map operation on ds
+  ds = ds->Map({random_apply}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (Probability has to be between 0 and 1)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestRandomApplyFail4) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail4 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
+
+  // RandomApply: transform list must not be empty
+  std::vector<std::shared_ptr<TensorTransform>> list = {};
+  auto random_apply = transforms::RandomApply(list);
+
+  // Create a Map operation on ds
+  ds = ds->Map({random_apply}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (transform list must not be empty)
+  EXPECT_EQ(iter, nullptr);
 }
 
 TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
@@ -360,11 +470,11 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
   std::string folder_path = datasets_root_path_ + "/testPK/data/";
   std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
   EXPECT_NE(ds, nullptr);
-  /* FIXME - Disable until proper external API for RandomChoice is provided
+
   // Create objects for the tensor ops
-  std::shared_ptr<TensorOperation> random_choice =
-    transforms::RandomChoice({vision::Resize({777, 777}), vision::Resize({888, 888})});
-  EXPECT_NE(random_choice, nullptr);
+  std::shared_ptr<TensorTransform> resize_op1(new vision::Resize({777, 777}));
+  std::shared_ptr<TensorTransform> resize_op2(new vision::Resize({888, 888}));
+  auto random_choice = transforms::RandomChoice({resize_op1, resize_op2});
 
   // Create a Map operation on ds
   ds = ds->Map({random_choice}, {"image"});
@@ -393,25 +503,71 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
 
   // Manually terminate the pipeline
   iter->Stop();
-  */
 }
 
-TEST_F(MindDataTestPipeline, TestRandomChoiceFail) {
-  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail with invalid transform.";
-  /* FIXME - Disable until proper external API for RandomChoice is provided
+TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail1 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
+
   // Resize: Non-positive size value: -1 at element: 0
   // RandomChoice: transform ops must not be null
-  std::shared_ptr<TensorOperation> random_choice1 = transforms::RandomChoice({vision::Decode(), vision::Resize({-1})});
-  EXPECT_EQ(random_choice1, nullptr);
+  auto decode_op = vision::Decode();
+  auto resize_op = vision::Resize({-1});
+  auto random_choice = transforms::RandomChoice({decode_op, resize_op});
+
+  // Create a Map operation on ds
+  ds = ds->Map({random_choice}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (transform ops must not be null)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail2 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
 
   // RandomChoice: transform ops must not be null
-  std::shared_ptr<TensorOperation> random_choice2 = transforms::RandomChoice({vision::Decode(), nullptr});
-  EXPECT_EQ(random_choice2, nullptr);
+  std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
+  std::shared_ptr<TensorTransform> random_choice(new transforms::RandomApply({decode_op, nullptr}));
+
+  // Create a Map operation on ds
+  ds = ds->Map({random_choice}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (transform ops must not be null)
+  EXPECT_EQ(iter, nullptr);
+}
+
+TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail3 with invalid transform.";
+
+  // Create a Cifar10 Dataset
+  std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
+  std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
+  EXPECT_NE(ds, nullptr);
 
   // RandomChoice: transform list must not be empty
-  std::shared_ptr<TensorOperation> random_choice3 = transforms::RandomChoice({});
-  EXPECT_EQ(random_choice3, nullptr);
-  */
+  std::vector<std::shared_ptr<TensorTransform>> list = {};
+  auto random_choice = transforms::RandomChoice(list);
+
+  // Create a Map operation on ds
+  ds = ds->Map({random_choice}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  // Expect failure: invalid RandomApply parameter (transform list must not be empty)
+  EXPECT_EQ(iter, nullptr);
 }
 
 TEST_F(MindDataTestPipeline, TestTypeCastSuccess) {

tests/ut/cpp/dataset/optimization_pass_test.cc

@@ -27,6 +27,8 @@
 #include "minddata/dataset/include/transforms.h"
 #include "minddata/dataset/include/vision.h"
 #include "minddata/dataset/include/vision_lite.h"
+#include "minddata/dataset/kernels/ir/data/transforms_ir.h"
+#include "minddata/dataset/kernels/ir/vision/vision_ir.h"
 
 using namespace mindspore::dataset;
 using mindspore::LogStream;
@@ -42,8 +44,7 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestAutoWorkerPass) {
   std::shared_ptr<Dataset> map_leaf = ImageFolder("dir")->SetNumWorkers(0);
   std::shared_ptr<Dataset> nonmap_leaf = RandomData(44, schema)->SetNumWorkers(0);
   std::shared_ptr<Dataset> batch = Zip({map_leaf, nonmap_leaf})->Batch(1)->SetNumWorkers(0);
-  /* FIXME - Will uncomment out when full external API support is provided
-  std::shared_ptr<Dataset> map = batch->Map({})->SetNumWorkers(0);
+  std::shared_ptr<Dataset> map = batch->Map({std::shared_ptr<TensorTransform>()})->SetNumWorkers(0);
   //  {ImageFolder, RandomData} -> zip -> batch
   EXPECT_EQ(map_leaf->IRNode()->num_workers(), 0);
   EXPECT_EQ(nonmap_leaf->IRNode()->num_workers(), 0);
@@ -65,15 +66,14 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestAutoWorkerPass) {
   MS_LOG(DEBUG) << nonmap_leaf->IRNode()->Name() << ": num_worker=" << nonmap_leaf->IRNode()->num_workers();
   MS_LOG(DEBUG) << batch->IRNode()->Name() << ": num_worker=" << batch->IRNode()->num_workers();
   MS_LOG(DEBUG) << map->IRNode()->Name() << ": num_worker=" << map->IRNode()->num_workers();
-  */
 }
 
 TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPass) {
   MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPass.";
   std::string folder_path = datasets_root_path_ + "/testPK/data/";
-  /* FIXME - Will uncomment out when full external API support is provided
-  std::shared_ptr<Dataset> root =
-    ImageFolder(folder_path, false)->Map({vision::Decode(), vision::RandomResizedCrop({100})}, {"image"});
+  auto decode_op = vision::Decode();
+  auto random_resized_crop_op = vision::RandomResizedCrop({100});
+  std::shared_ptr<Dataset> root = ImageFolder(folder_path, false)->Map({decode_op, random_resized_crop_op}, {"image"});
 
   TensorOpFusionPass fusion_pass;
   bool modified = false;
@@ -85,27 +85,26 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPass) {
   auto fused_ops = map_node->operations();
   ASSERT_EQ(fused_ops.size(), 1);
   ASSERT_EQ(fused_ops[0]->Name(), vision::kRandomCropDecodeResizeOperation);
-  */
 }
 
 TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPassPreBuiltTensorOperation) {
   MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPassPreBuiltTensorOperation.";
   std::string folder_path = datasets_root_path_ + "/testPK/data/";
-  /* FIXME - Will uncomment out when full external API support is provided
   // make prebuilt tensor operation
-  auto decode = std::make_shared<transforms::PreBuiltOperation>(vision::Decode()->Build());
-  auto resize = std::make_shared<transforms::PreBuiltOperation>(vision::RandomResizedCrop({100})->Build());
-  std::shared_ptr<Dataset> root = ImageFolder(folder_path, false)->Map({decode, resize}, {"image"});
+  auto decode = std::make_shared<transforms::PreBuiltOperation>(vision::Decode().Parse()->Build());
+  auto resize = std::make_shared<transforms::PreBuiltOperation>(vision::RandomResizedCrop({100}).Parse()->Build());
+  std::vector<std::shared_ptr<TensorOperation>> op_list = {decode, resize};
+  std::vector<std::string> op_name = {"image"};
+  std::shared_ptr<DatasetNode> root = ImageFolder(folder_path, false)->IRNode();
+  std::shared_ptr<MapNode> map_node = std::make_shared<MapNode>(root, op_list, op_name);
 
   TensorOpFusionPass fusion_pass;
   bool modified = false;
-  std::shared_ptr<MapNode> map_node = std::dynamic_pointer_cast<MapNode>(root->IRNode());
   // no deepcopy is performed because this doesn't go through tree_adapter
-  fusion_pass.Run(root->IRNode(), &modified);
+  fusion_pass.Run(map_node, &modified);
   EXPECT_EQ(modified, true);
   ASSERT_NE(map_node, nullptr);
   auto fused_ops = map_node->operations();
   ASSERT_EQ(fused_ops.size(), 1);
   ASSERT_EQ(fused_ops[0]->Name(), kRandomCropDecodeResizeOp);
-  */
 }