diff --git a/mindspore/ccsrc/minddata/dataset/api/transforms.cc b/mindspore/ccsrc/minddata/dataset/api/transforms.cc index 46822e1323..a21bf34724 100644 --- a/mindspore/ccsrc/minddata/dataset/api/transforms.cc +++ b/mindspore/ccsrc/minddata/dataset/api/transforms.cc @@ -17,39 +17,168 @@ #include "minddata/dataset/include/transforms.h" // Kernel data headers (in alphabetical order) +#include "minddata/dataset/kernels/data/compose_op.h" +#include "minddata/dataset/kernels/data/duplicate_op.h" #include "minddata/dataset/kernels/data/one_hot_op.h" +#include "minddata/dataset/kernels/data/random_apply_op.h" +#include "minddata/dataset/kernels/data/random_choice_op.h" #include "minddata/dataset/kernels/data/type_cast_op.h" +#ifndef ENABLE_ANDROID +#include "minddata/dataset/kernels/data/unique_op.h" +#endif namespace mindspore { namespace dataset { TensorOperation::TensorOperation() {} +/* ####################################### Validator Functions ############################################ */ +Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector &fill_value) { + if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) { + std::string err_msg = + transform_name + ": fill_value vector has incorrect size: " + std::to_string(fill_value.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (uint8_t single_fill_value : fill_value) { + if (single_fill_value > 255) { + std::string err_msg = + transform_name + ": fill_value has to be between 0 and 255, got:" + std::to_string(single_fill_value); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + } + + return Status::OK(); +} + +Status ValidateProbability(const std::string &transform_name, const float &probability) { + if (probability < 0.0 || probability > 1.0) { + std::string err_msg = + transform_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + return Status::OK(); +} + +Status ValidateVectorPadding(const std::string &transform_name, const std::vector &padding) { + if (padding.empty() || padding.size() == 3 || padding.size() > 4) { + std::string err_msg = transform_name + ": padding vector has incorrect size: " + std::to_string(padding.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (int32_t i = 0; i < padding.size(); ++i) { + if (padding[i] < 0) { + std::string err_msg = + transform_name + + ": invalid padding, padding value must be greater than or equal to 0, got: " + std::to_string(padding[i]); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (padding[i] == INT_MAX) { + std::string err_msg = + transform_name + ": invalid padding, padding value too large, got: " + std::to_string(padding[i]); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + } + + return Status::OK(); +} + +Status ValidateVectorPositive(const std::string &transform_name, const std::vector &size) { + for (int32_t i = 0; i < size.size(); ++i) { + if (size[i] <= 0) { + std::string err_msg = + transform_name + ": Non-positive size value: " + std::to_string(size[i]) + " at element: " + std::to_string(i); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + } + + return Status::OK(); +} + +Status ValidateVectorTransforms(const std::string &transform_name, + const std::vector> &transforms) { + if (transforms.empty()) { + std::string err_msg = transform_name + ": transform list must not be empty."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (int32_t i = 0; i < transforms.size(); ++i) { + if (transforms[i] == nullptr) { + std::string err_msg = + transform_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + } + + return Status::OK(); +} + +bool CmpFloat(const float &a, const float &b, float epsilon) { return (std::fabs(a - b) < epsilon); } + // Transform operations for data. namespace transforms { // FUNCTIONS TO CREATE DATA TRANSFORM OPERATIONS // (In alphabetical order) +// Function to create ComposeOperation. +std::shared_ptr Compose(const std::vector> &transforms) { + auto op = std::make_shared(transforms); + // Input validation + return op->ValidateParams() ? op : nullptr; +} + +// Function to create DuplicateOperation. +std::shared_ptr Duplicate() { + auto op = std::make_shared(); + // Input validation + return op->ValidateParams() ? op : nullptr; +} + // Function to create OneHotOperation. std::shared_ptr OneHot(int32_t num_classes) { auto op = std::make_shared(num_classes); // Input validation - if (!op->ValidateParams()) { - return nullptr; - } - return op; + return op->ValidateParams() ? op : nullptr; +} + +// Function to create RandomApplyOperation. +std::shared_ptr RandomApply(const std::vector> &transforms, + double prob) { + auto op = std::make_shared(transforms, prob); + // Input validation + return op->ValidateParams() ? op : nullptr; +} + +// Function to create RandomChoiceOperation. +std::shared_ptr RandomChoice(const std::vector> &transforms) { + auto op = std::make_shared(transforms); + // Input validation + return op->ValidateParams() ? op : nullptr; } // Function to create TypeCastOperation. std::shared_ptr TypeCast(std::string data_type) { auto op = std::make_shared(data_type); // Input validation - if (!op->ValidateParams()) { - return nullptr; - } - return op; + return op->ValidateParams() ? op : nullptr; +} + +#ifndef ENABLE_ANDROID +// Function to create UniqueOperation. +std::shared_ptr Unique() { + auto op = std::make_shared(); + // Input validation + return op->ValidateParams() ? op : nullptr; } +#endif /* ####################################### Validator Functions ############################################ */ @@ -57,13 +186,33 @@ std::shared_ptr TypeCast(std::string data_type) { // (In alphabetical order) +// ComposeOperation +ComposeOperation::ComposeOperation(const std::vector> &transforms) + : transforms_(transforms) {} + +Status ComposeOperation::ValidateParams() { + RETURN_IF_NOT_OK(ValidateVectorTransforms("Compose", transforms_)); + return Status::OK(); +} + +std::shared_ptr ComposeOperation::Build() { + std::vector> tensor_ops; + (void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops), + [](std::shared_ptr op) -> std::shared_ptr { return op->Build(); }); + return std::make_shared(tensor_ops); +} + +// DuplicateOperation +Status DuplicateOperation::ValidateParams() { return Status::OK(); } + +std::shared_ptr DuplicateOperation::Build() { return std::make_shared(); } + // OneHotOperation OneHotOperation::OneHotOperation(int32_t num_classes) : num_classes_(num_classes) {} Status OneHotOperation::ValidateParams() { if (num_classes_ <= 0) { - std::string err_msg = - "OneHot: Number of classes must be greater than 0. num_classes: " + std::to_string(num_classes_); + std::string err_msg = "OneHot: Number of classes must be greater than 0, but got: " + std::to_string(num_classes_); MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } @@ -73,6 +222,39 @@ Status OneHotOperation::ValidateParams() { std::shared_ptr OneHotOperation::Build() { return std::make_shared(num_classes_); } +// RandomApplyOperation +RandomApplyOperation::RandomApplyOperation(const std::vector> &transforms, double prob) + : transforms_(transforms), prob_(prob) {} + +Status RandomApplyOperation::ValidateParams() { + RETURN_IF_NOT_OK(ValidateVectorTransforms("RandomApply", transforms_)); + RETURN_IF_NOT_OK(ValidateProbability("RandomApply", prob_)); + return Status::OK(); +} + +std::shared_ptr RandomApplyOperation::Build() { + std::vector> tensor_ops; + (void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops), + [](std::shared_ptr op) -> std::shared_ptr { return op->Build(); }); + return std::make_shared(prob_, tensor_ops); +} + +// RandomChoiceOperation +RandomChoiceOperation::RandomChoiceOperation(const std::vector> &transforms) + : transforms_(transforms) {} + +Status RandomChoiceOperation::ValidateParams() { + RETURN_IF_NOT_OK(ValidateVectorTransforms("RandomChoice", transforms_)); + return Status::OK(); +} + +std::shared_ptr RandomChoiceOperation::Build() { + std::vector> tensor_ops; + (void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops), + [](std::shared_ptr op) -> std::shared_ptr { return op->Build(); }); + return std::make_shared(tensor_ops); +} + // TypeCastOperation TypeCastOperation::TypeCastOperation(std::string data_type) : data_type_(data_type) {} @@ -83,7 +265,7 @@ Status TypeCastOperation::ValidateParams() { if (itr == predefine_type.end()) { std::string err_msg = "TypeCast: Invalid data type: " + data_type_; MS_LOG(ERROR) << "TypeCast: Only supports data type bool, int8, uint8, int16, uint16, int32, uint32, " - << "int64, uint64, float16, float32, float64, string, but got " << data_type_; + << "int64, uint64, float16, float32, float64, string, but got: " << data_type_; RETURN_STATUS_SYNTAX_ERROR(err_msg); } @@ -92,6 +274,13 @@ Status TypeCastOperation::ValidateParams() { std::shared_ptr TypeCastOperation::Build() { return std::make_shared(data_type_); } +#ifndef ENABLE_ANDROID +// UniqueOperation +Status UniqueOperation::ValidateParams() { return Status::OK(); } + +std::shared_ptr UniqueOperation::Build() { return std::make_shared(); } +#endif + } // namespace transforms } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/vision.cc b/mindspore/ccsrc/minddata/dataset/api/vision.cc index d7d354df43..3a9685cf87 100644 --- a/mindspore/ccsrc/minddata/dataset/api/vision.cc +++ b/mindspore/ccsrc/minddata/dataset/api/vision.cc @@ -394,77 +394,7 @@ std::shared_ptr UniformAugment(std::vectorValidateParams() ? op : nullptr; } - #endif -/* ####################################### Validator Functions ############################################ */ -Status ValidateVectorFillvalue(const std::string &dataset_name, const std::vector &fill_value) { - if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) { - std::string err_msg = dataset_name + ": fill_value vector has incorrect size: " + std::to_string(fill_value.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - for (uint8_t single_fill_value : fill_value) { - if (single_fill_value > 255) { - std::string err_msg = - dataset_name + ": fill_value has to be between 0 and 255, got:" + std::to_string(single_fill_value); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - } - - return Status::OK(); -} - -Status ValidateVectorProbability(const std::string &dataset_name, const float &probability) { - if (probability < 0.0 || probability > 1.0) { - std::string err_msg = - dataset_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - - return Status::OK(); -} - -Status ValidateVectorPadding(const std::string &dataset_name, const std::vector &padding) { - if (padding.empty() || padding.size() == 3 || padding.size() > 4) { - std::string err_msg = dataset_name + ": padding vector has incorrect size: " + std::to_string(padding.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - for (int32_t i = 0; i < padding.size(); ++i) { - if (padding[i] < 0) { - std::string err_msg = - dataset_name + - ": invalid padding, padding value must be greater than or equal to 0, got: " + std::to_string(padding[i]); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - if (padding[i] == INT_MAX) { - std::string err_msg = - dataset_name + ": invalid padding, padding value too large, got: " + std::to_string(padding[i]); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - } - - return Status::OK(); -} - -Status ValidateVectorPositive(const std::string &dataset_name, const std::vector &size) { - for (int32_t i = 0; i < size.size(); ++i) { - if (size[i] <= 0) { - std::string err_msg = - dataset_name + ": Non-positive size value: " + std::to_string(size[i]) + " at element: " + std::to_string(i); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - } - - return Status::OK(); -} - -bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f) { return (std::fabs(a - b) < epsilon); } /* ####################################### Derived TensorOperation classes ################################# */ @@ -503,17 +433,8 @@ BoundingBoxAugmentOperation::BoundingBoxAugmentOperation(std::shared_ptr 1.0) { - std::string err_msg = "BoundingBoxAugment: ratio has to be between 0.0 and 1.0, got: " + std::to_string(ratio_); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } + RETURN_IF_NOT_OK(ValidateVectorTransforms("BoundingBoxAugment", {transform_})); + RETURN_IF_NOT_OK(ValidateProbability("BoundingBoxAugment", ratio_)); return Status::OK(); } @@ -1315,7 +1236,7 @@ std::shared_ptr RandomCropWithBBoxOperation::Build() { RandomHorizontalFlipOperation::RandomHorizontalFlipOperation(float probability) : probability_(probability) {} Status RandomHorizontalFlipOperation::ValidateParams() { - RETURN_IF_NOT_OK(ValidateVectorProbability("RandomHorizontalFlip", probability_)); + RETURN_IF_NOT_OK(ValidateProbability("RandomHorizontalFlip", probability_)); return Status::OK(); } @@ -1330,7 +1251,7 @@ RandomHorizontalFlipWithBBoxOperation::RandomHorizontalFlipWithBBoxOperation(flo : probability_(probability) {} Status RandomHorizontalFlipWithBBoxOperation::ValidateParams() { - RETURN_IF_NOT_OK(ValidateVectorProbability("RandomHorizontalFlipWithBBox", probability_)); + RETURN_IF_NOT_OK(ValidateProbability("RandomHorizontalFlipWithBBox", probability_)); return Status::OK(); } @@ -1696,7 +1617,7 @@ std::shared_ptr RandomSolarizeOperation::Build() { RandomVerticalFlipOperation::RandomVerticalFlipOperation(float probability) : probability_(probability) {} Status RandomVerticalFlipOperation::ValidateParams() { - RETURN_IF_NOT_OK(ValidateVectorProbability("RandomVerticalFlip", probability_)); + RETURN_IF_NOT_OK(ValidateProbability("RandomVerticalFlip", probability_)); return Status::OK(); } @@ -1711,7 +1632,7 @@ RandomVerticalFlipWithBBoxOperation::RandomVerticalFlipWithBBoxOperation(float p : probability_(probability) {} Status RandomVerticalFlipWithBBoxOperation::ValidateParams() { - RETURN_IF_NOT_OK(ValidateVectorProbability("RandomVerticalFlipWithBBox", probability_)); + RETURN_IF_NOT_OK(ValidateProbability("RandomVerticalFlipWithBBox", probability_)); return Status::OK(); } @@ -1945,21 +1866,15 @@ UniformAugOperation::UniformAugOperation(std::vector transforms_.size()) { - std::string err_msg = "UniformAug: num_ops is greater than transforms size, num_ops: " + std::to_string(num_ops_); + std::string err_msg = "UniformAug: num_ops is greater than transforms size, but got: " + std::to_string(num_ops_); MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } - for (int32_t i = 0; i < transforms_.size(); ++i) { - if (transforms_[i] == nullptr) { - std::string err_msg = "UniformAug: transform ops must not be null."; - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - } // num_ops if (num_ops_ <= 0) { - std::string err_msg = "UniformAug: num_ops must be greater than 0, num_ops: " + std::to_string(num_ops_); + std::string err_msg = "UniformAug: num_ops must be greater than 0, but got: " + std::to_string(num_ops_); MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } diff --git a/mindspore/ccsrc/minddata/dataset/include/transforms.h b/mindspore/ccsrc/minddata/dataset/include/transforms.h index 2df71e40be..39977ded67 100644 --- a/mindspore/ccsrc/minddata/dataset/include/transforms.h +++ b/mindspore/ccsrc/minddata/dataset/include/transforms.h @@ -44,12 +44,50 @@ class TensorOperation : public std::enable_shared_from_this { virtual Status ValidateParams() = 0; }; +// Helper function to validate fill value +Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector &fill_value); + +// Helper function to validate probability +Status ValidateProbability(const std::string &transform_name, const float &probability); + +// Helper function to validate padding +Status ValidateVectorPadding(const std::string &transform_name, const std::vector &padding); + +// Helper function to validate size +Status ValidateVectorPositive(const std::string &transform_name, const std::vector &size); + +// Helper function to validate transforms +Status ValidateVectorTransforms(const std::string &transform_name, + const std::vector> &transforms); + +// Helper function to compare float value +bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f); + // Transform operations for performing data transformation. namespace transforms { // Transform Op classes (in alphabetical order) +class ComposeOperation; +class DuplicateOperation; class OneHotOperation; +class RandomApplyOperation; +class RandomChoiceOperation; class TypeCastOperation; +#ifndef ENABLE_ANDROID +class UniqueOperation; +#endif + +/// \brief Function to create a Compose TensorOperation. +/// \notes Compose a list of transforms into a single transform. +/// \param[in] transforms A vector of transformations to be applied. +/// \return Shared pointer to the current TensorOperation. +std::shared_ptr Compose(const std::vector> &transforms); + +/// \brief Function to create a Duplicate TensorOperation. +/// \notes Duplicate the input tensor to a new output tensor. +/// The input tensor is carried over to the output list. +/// \return Shared pointer to the current TensorOperation. +std::shared_ptr Duplicate(); /// \brief Function to create a OneHot TensorOperation. /// \notes Convert the labels into OneHot format. @@ -57,14 +95,61 @@ class TypeCastOperation; /// \return Shared pointer to the current TensorOperation. std::shared_ptr OneHot(int32_t num_classes); +/// \brief Function to create a RandomApply TensorOperation. +/// \notes Randomly perform a series of transforms with a given probability. +/// \param[in] transforms A vector of transformations to be applied. +/// \param[in] prob The probability to apply the transformation list (default=0.5) +/// \return Shared pointer to the current TensorOperation. +std::shared_ptr RandomApply(const std::vector> &transforms, + double prob = 0.5); + +/// \brief Function to create a RandomChoice TensorOperation. +/// \notes Randomly selects one transform from a list of transforms to perform operation. +/// \param[in] transforms A vector of transformations to be chosen from to apply. +/// \return Shared pointer to the current TensorOperation. +std::shared_ptr RandomChoice(const std::vector> &transforms); + /// \brief Function to create a TypeCast TensorOperation. /// \notes Tensor operation to cast to a given MindSpore data type. /// \param[in] data_type mindspore.dtype to be cast to. /// \return Shared pointer to the current TensorOperation. std::shared_ptr TypeCast(std::string data_type); +#ifndef ENABLE_ANDROID +/// \brief Function to create a Unique TensorOperation. +/// \notes Return an output tensor containing all the unique elements of the input tensor in +/// the same order that they occur in the input tensor. +/// \return Shared pointer to the current TensorOperation. +std::shared_ptr Unique(); +#endif + /* ####################################### Derived TensorOperation classes ################################# */ +class ComposeOperation : public TensorOperation { + public: + explicit ComposeOperation(const std::vector> &transforms); + + ~ComposeOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + private: + std::vector> transforms_; +}; + +class DuplicateOperation : public TensorOperation { + public: + DuplicateOperation() = default; + + ~DuplicateOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; +}; + class OneHotOperation : public TensorOperation { public: explicit OneHotOperation(int32_t num_classes_); @@ -79,6 +164,35 @@ class OneHotOperation : public TensorOperation { float num_classes_; }; +class RandomApplyOperation : public TensorOperation { + public: + explicit RandomApplyOperation(const std::vector> &transforms, double prob); + + ~RandomApplyOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + private: + std::vector> transforms_; + double prob_; +}; + +class RandomChoiceOperation : public TensorOperation { + public: + explicit RandomChoiceOperation(const std::vector> &transforms); + + ~RandomChoiceOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + private: + std::vector> transforms_; +}; + class TypeCastOperation : public TensorOperation { public: explicit TypeCastOperation(std::string data_type); @@ -92,6 +206,19 @@ class TypeCastOperation : public TensorOperation { private: std::string data_type_; }; + +#ifndef ENABLE_ANDROID +class UniqueOperation : public TensorOperation { + public: + UniqueOperation() = default; + + ~UniqueOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; +}; +#endif } // namespace transforms } // namespace dataset } // namespace mindspore diff --git a/tests/ut/cpp/dataset/c_api_transforms_test.cc b/tests/ut/cpp/dataset/c_api_transforms_test.cc index c0e96eeca6..0719d386e5 100644 --- a/tests/ut/cpp/dataset/c_api_transforms_test.cc +++ b/tests/ut/cpp/dataset/c_api_transforms_test.cc @@ -28,6 +28,107 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { // Tests for data transforms ops (in alphabetical order) +TEST_F(MindDataTestPipeline, TestComposeSuccess) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeSuccess."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 3)); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr compose = transforms::Compose({vision::Decode(), vision::Resize({777, 777})}); + EXPECT_NE(compose, nullptr); + + // Create a Map operation on ds + ds = ds->Map({compose}, {"image"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Label shape: " << label->shape(); + EXPECT_EQ(image->shape()[0], 777); + EXPECT_EQ(image->shape()[1], 777); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestComposeFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail with invalid transform."; + + // Resize: Non-positive size value: -1 at element: 0 + // Compose: transform ops must not be null + std::shared_ptr compose1 = transforms::Compose({vision::Decode(), vision::Resize({-1})}); + EXPECT_EQ(compose1, nullptr); + + // Compose: transform ops must not be null + std::shared_ptr compose2 = transforms::Compose({vision::Decode(), nullptr}); + EXPECT_EQ(compose2, nullptr); + + // Compose: transform list must not be empty + std::shared_ptr compose3 = transforms::Compose({}); + EXPECT_EQ(compose3, nullptr); +} + +TEST_F(MindDataTestPipeline, TestDuplicateSuccess) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDuplicateSuccess."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr duplicate = transforms::Duplicate(); + EXPECT_NE(duplicate, nullptr); + + // Create a Map operation on ds + ds = ds->Map({duplicate}, {"image"}, {"image", "image_copy"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + auto image_copy = row["image_copy"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + EXPECT_EQ(*image, *image_copy); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 10); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestOneHotSuccess1) { // Testing CutMixBatch on a batch of CHW images // Create a Cifar10 Dataset @@ -157,6 +258,127 @@ TEST_F(MindDataTestPipeline, TestOneHotFail) { EXPECT_EQ(one_hot_op2, nullptr); } +TEST_F(MindDataTestPipeline, TestRandomApplySuccess) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplySuccess."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr random_apply = transforms::RandomApply({vision::Resize({777, 777})}, 0.8); + EXPECT_NE(random_apply, nullptr); + + // Create a Map operation on ds + ds = ds->Map({random_apply}, {"image"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Label shape: " << label->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 5); + + // Manually terminate the pipeline + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestRandomApplyFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail with invalid transform."; + + // Resize: Non-positive size value: -1 at element: 0 + // RandomApply: transform ops must not be null + std::shared_ptr random_apply1 = transforms::RandomApply({vision::Decode(), vision::Resize({-1})}); + EXPECT_EQ(random_apply1, nullptr); + + // RandomApply: transform ops must not be null + std::shared_ptr random_apply2 = transforms::RandomApply({vision::Decode(), nullptr}); + EXPECT_EQ(random_apply2, nullptr); + + // RandomApply: transform list must not be empty + std::shared_ptr random_apply3 = transforms::RandomApply({}); + EXPECT_EQ(random_apply3, nullptr); + + // RandomApply: Probability has to be between 0 and 1 + std::shared_ptr random_apply4 = transforms::RandomApply({vision::Resize({100})}, -1); + EXPECT_EQ(random_apply4, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceSuccess."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr random_choice = + transforms::RandomChoice({vision::Resize({777, 777}), vision::Resize({888, 888})}); + EXPECT_NE(random_choice, nullptr); + + // Create a Map operation on ds + ds = ds->Map({random_choice}, {"image"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Label shape: " << label->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestRandomChoiceFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail with invalid transform."; + + // Resize: Non-positive size value: -1 at element: 0 + // RandomChoice: transform ops must not be null + std::shared_ptr random_choice1 = transforms::RandomChoice({vision::Decode(), vision::Resize({-1})}); + EXPECT_EQ(random_choice1, nullptr); + + // RandomChoice: transform ops must not be null + std::shared_ptr random_choice2 = transforms::RandomChoice({vision::Decode(), nullptr}); + EXPECT_EQ(random_choice2, nullptr); + + // RandomChoice: transform list must not be empty + std::shared_ptr random_choice3 = transforms::RandomChoice({}); + EXPECT_EQ(random_choice3, nullptr); +} + TEST_F(MindDataTestPipeline, TestTypeCastSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTypeCastSuccess."; diff --git a/tests/ut/cpp/dataset/c_api_vision_test.cc b/tests/ut/cpp/dataset/c_api_vision_test.cc index d13b3236fa..a5306e4aea 100644 --- a/tests/ut/cpp/dataset/c_api_vision_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_test.cc @@ -2799,14 +2799,8 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegFail) { EXPECT_EQ(soft_dvpp_decode_resize_jpeg_op4, nullptr); } -TEST_F(MindDataTestPipeline, DISABLED_TestUniformAugmentFail1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail1 with invalid zero num_ops parameter."; - - // Create a Mnist Dataset - std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 20)); - EXPECT_NE(ds, nullptr); - +TEST_F(MindDataTestPipeline, TestUniformAugmentFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail1 with invalid num_ops parameter."; // Create objects for the tensor ops std::shared_ptr random_crop_op = vision::RandomCrop({28, 28}); EXPECT_NE(random_crop_op, nullptr); @@ -2814,29 +2808,33 @@ TEST_F(MindDataTestPipeline, DISABLED_TestUniformAugmentFail1) { std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); EXPECT_NE(center_crop_op, nullptr); - // Try UniformAugment with invalid zero num_ops value - std::shared_ptr uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 0); - EXPECT_EQ(uniform_aug_op, nullptr); -} + // UniformAug: num_ops must be greater than 0 + std::shared_ptr uniform_aug_op1 = vision::UniformAugment({random_crop_op, center_crop_op}, 0); + EXPECT_EQ(uniform_aug_op1, nullptr); -TEST_F(MindDataTestPipeline, DISABLED_TestUniformAugmentFail2) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail2 with invalid negative num_ops parameter."; + // UniformAug: num_ops must be greater than 0 + std::shared_ptr uniform_aug_op2 = vision::UniformAugment({random_crop_op, center_crop_op}, -1); + EXPECT_EQ(uniform_aug_op2, nullptr); - // Create a Mnist Dataset - std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 20)); - EXPECT_NE(ds, nullptr); + // UniformAug: num_ops is greater than transforms size + std::shared_ptr uniform_aug_op3 = vision::UniformAugment({random_crop_op, center_crop_op}, 3); + EXPECT_EQ(uniform_aug_op3, nullptr); +} - // Create objects for the tensor ops - std::shared_ptr random_crop_op = vision::RandomCrop({28, 28}); - EXPECT_NE(random_crop_op, nullptr); +TEST_F(MindDataTestPipeline, TestUniformAugmentFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail2 with invalid transform."; - std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); - EXPECT_NE(center_crop_op, nullptr); + // UniformAug: transform ops must not be null + std::shared_ptr uniform_aug_op1 = vision::UniformAugment({vision::RandomCrop({-28})}, 1); + EXPECT_EQ(uniform_aug_op1, nullptr); + + // UniformAug: transform ops must not be null + std::shared_ptr uniform_aug_op2 = vision::UniformAugment({vision::RandomCrop({28}), nullptr}, 2); + EXPECT_EQ(uniform_aug_op2, nullptr); - // Try UniformAugment with invalid negative num_ops value - std::shared_ptr uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, -1); - EXPECT_EQ(uniform_aug_op, nullptr); + // UniformAug: transform list must not be empty + std::shared_ptr uniform_aug_op3 = vision::UniformAugment({}, 1); + EXPECT_EQ(uniform_aug_op3, nullptr); } TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {