diff --git a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt index 244ead5e11..c6f1f39334 100644 --- a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt @@ -96,6 +96,7 @@ add_dependencies(cpp-API core) add_dependencies(engine-ir-datasetops core) add_dependencies(engine-ir-datasetops-source core) add_dependencies(engine-ir-cache core) +add_dependencies(kernels-ir core) if(ENABLE_ACL) add_dependencies(kernels-dvpp-image core dvpp-utils) @@ -144,6 +145,7 @@ set(submodules $ $ $ + $ ) if(ENABLE_ACL) diff --git a/mindspore/ccsrc/minddata/dataset/api/transforms.cc b/mindspore/ccsrc/minddata/dataset/api/transforms.cc index 1be01ecab8..9491a0e4d1 100644 --- a/mindspore/ccsrc/minddata/dataset/api/transforms.cc +++ b/mindspore/ccsrc/minddata/dataset/api/transforms.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "minddata/dataset/include/transforms.h" +#include "minddata/dataset/kernels/ir/validators.h" // Kernel data headers (in alphabetical order) #include "minddata/dataset/kernels/data/compose_op.h" @@ -30,180 +31,6 @@ namespace mindspore { namespace dataset { -/* ####################################### Validator Functions ############################################ */ -Status ValidateProbability(const std::string &op_name, const float probability) { - if (probability < 0.0 || probability > 1.0) { - std::string err_msg = op_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - - return Status::OK(); -} - -Status ValidateIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, true)); - return Status::OK(); -} - -Status ValidateFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, true)); - return Status::OK(); -} - -Status ValidateVectorFillvalue(const std::string &op_name, const std::vector &fill_value) { - if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) { - std::string err_msg = - op_name + ": fill_value expecting size 1 or 3, got fill_value.size(): " + std::to_string(fill_value.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - // Note that fill_value need to be in range [0, 255], - // but we omit the check since its type is uint8_t - return Status::OK(); -} - -Status ValidateVectorColorAttribute(const std::string &op_name, const std::string &attr_name, - const std::vector &attr, const std::vector &range) { - if (attr.empty() || attr.size() > 2) { - std::string err_msg = op_name + ":" + attr_name + " expecting size 1 or 2, but got: " + std::to_string(attr.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - for (auto &attr_val : attr) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, attr_name, attr_val, range, false, false)); - } - if (attr.size() == 2 && (attr[0] > attr[1])) { - std::string err_msg = op_name + ":" + attr_name + - " lower bound must be less or equal to upper bound, got lb: " + std::to_string(attr[0]) + - ", ub: " + std::to_string(attr[1]); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - - return Status::OK(); -} - -Status ValidateVectorMeanStd(const std::string &op_name, const std::vector &mean, - const std::vector &std) { - if (mean.size() != 3) { - std::string err_msg = op_name + ": mean expecting size 3, got size: " + std::to_string(mean.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - if (std.size() != 3) { - std::string err_msg = op_name + ": std expecting size 3, got size: " + std::to_string(std.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - // check std/mean value - for (int32_t i = 0; i < std.size(); ++i) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, "mean", mean[i], {0.0, 255.0}, false, false)); - RETURN_IF_NOT_OK(ValidateScalar(op_name, "std", std[i], {0.0, 255.0}, true, false)); - } - - return Status::OK(); -} - -Status ValidateVectorPadding(const std::string &op_name, const std::vector &padding) { - if (padding.empty() || padding.size() == 3 || padding.size() > 4) { - std::string err_msg = op_name + ": padding expecting size 1, 2 or 4, got size: " + std::to_string(padding.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - for (const auto &pad_val : padding) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, "padding", pad_val, {0, INT_MAX}, false, false)); - } - - return Status::OK(); -} - -Status ValidateVectorPositive(const std::string &op_name, const std::string &vec_name, - const std::vector &vec) { - for (const auto &vec_val : vec) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, vec_name, vec_val, {0}, true)); - } - - return Status::OK(); -} - -Status ValidateVectorNonNegative(const std::string &op_name, const std::string &vec_name, - const std::vector &vec) { - for (const auto &vec_val : vec) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, vec_name, vec_val, {0}, false)); - } - - return Status::OK(); -} - -Status ValidateVectorSize(const std::string &op_name, const std::vector &size) { - if (size.empty() || size.size() > 2) { - std::string err_msg = op_name + ": size expecting size 2, got size.size(): " + std::to_string(size.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - for (const auto &size_val : size) { - RETURN_IF_NOT_OK(ValidateScalar(op_name, "size", size_val, {0, INT_MAX}, true, false)); - } - - return Status::OK(); -} - -Status ValidateVectorScale(const std::string &op_name, const std::vector &scale) { - if (scale.size() != 2) { - std::string err_msg = op_name + ": scale expecting size 2, got scale.size(): " + std::to_string(scale.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", scale[0], {0}, false)); - RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", scale[1], {0}, true)); - if (scale[1] < scale[0]) { - std::string err_msg = op_name + ": scale must be in the format of (min, max)."; - MS_LOG(ERROR) << op_name + ": scale must be in the format of (min, max), but got: " << scale; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - - return Status::OK(); -} - -Status ValidateVectorRatio(const std::string &op_name, const std::vector &ratio) { - if (ratio.size() != 2) { - std::string err_msg = op_name + ": ratio expecting size 2, got ratio.size(): " + std::to_string(ratio.size()); - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", ratio[0], {0}, true)); - RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", ratio[1], {0}, true)); - if (ratio[1] < ratio[0]) { - std::string err_msg = op_name + ": ratio must be in the format of (min, max)."; - MS_LOG(ERROR) << op_name + ": ratio must be in the format of (min, max), but got: " << ratio; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - - return Status::OK(); -} - -Status ValidateVectorTransforms(const std::string &op_name, - const std::vector> &transforms) { - if (transforms.empty()) { - std::string err_msg = op_name + ": transform list must not be empty."; - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - for (int32_t i = 0; i < transforms.size(); ++i) { - if (transforms[i] == nullptr) { - std::string err_msg = - op_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr."; - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - } - - return Status::OK(); -} - -bool CmpFloat(const float a, const float b, float epsilon) { return (std::fabs(a - b) < epsilon); } - // Transform operations for data. namespace transforms { diff --git a/mindspore/ccsrc/minddata/dataset/api/vision.cc b/mindspore/ccsrc/minddata/dataset/api/vision.cc index 4bfcf5a699..cfe874ca62 100644 --- a/mindspore/ccsrc/minddata/dataset/api/vision.cc +++ b/mindspore/ccsrc/minddata/dataset/api/vision.cc @@ -19,6 +19,8 @@ #ifndef ENABLE_ANDROID #include "minddata/dataset/kernels/image/image_utils.h" #endif +#include "minddata/dataset/kernels/ir/validators.h" + // Kernel image headers (in alphabetical order) #ifndef ENABLE_ANDROID #include "minddata/dataset/kernels/image/auto_contrast_op.h" diff --git a/mindspore/ccsrc/minddata/dataset/include/transforms.h b/mindspore/ccsrc/minddata/dataset/include/transforms.h index ebdecdf67b..22455b02e4 100644 --- a/mindspore/ccsrc/minddata/dataset/include/transforms.h +++ b/mindspore/ccsrc/minddata/dataset/include/transforms.h @@ -25,6 +25,9 @@ #include "include/api/status.h" #include "minddata/dataset/include/constants.h" +// (TEMPORARY) will be removed when Tensor op ir moved down +#include "minddata/dataset/kernels/ir/tensor_operation.h" + #ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ namespace nlohmann { @@ -45,8 +48,6 @@ using json = basic_json<>; namespace mindspore { namespace dataset { -class TensorOp; - // Char arrays storing name of corresponding classes (in alphabetical order) constexpr char kComposeOperation[] = "Compose"; constexpr char kDuplicateOperation[] = "Duplicate"; @@ -58,108 +59,6 @@ constexpr char kRandomSelectSubpolicyOperation[] = "RandomSelectSubpolicy"; constexpr char kTypeCastOperation[] = "TypeCast"; constexpr char kUniqueOperation[] = "Unique"; -// Abstract class to represent a dataset in the data pipeline. -class TensorOperation : public std::enable_shared_from_this { - public: - /// \brief Constructor - TensorOperation() : random_op_(false) {} - - /// \brief Constructor - explicit TensorOperation(bool random) : random_op_(random) {} - - /// \brief Destructor - ~TensorOperation() = default; - - /// \brief Pure virtual function to convert a TensorOperation class into a runtime TensorOp object. - /// \return shared pointer to the newly created TensorOp. - virtual std::shared_ptr Build() = 0; - - virtual Status ValidateParams() = 0; - - virtual std::string Name() const = 0; - - /// \brief Check whether the operation is deterministic. - /// \return true if this op is a random op (returns non-deterministic result e.g. RandomCrop) - bool IsRandomOp() const { return random_op_; } - - virtual Status to_json(nlohmann::json *out_json) { return Status::OK(); } - - protected: - bool random_op_; -}; - -// Helper function to validate probability -Status ValidateProbability(const std::string &op_name, const float probability); - -// Helper function to positive int scalar -Status ValidateIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar); - -// Helper function to positive float scalar -Status ValidateFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar); - -// Helper function to validate scalar -template -Status ValidateScalar(const std::string &op_name, const std::string &scalar_name, const T scalar, - const std::vector &range, bool left_open_interval = false, bool right_open_interval = false) { - if (range.empty() || range.size() > 2) { - std::string err_msg = "Range check expecting size 1 or 2, but got: " + std::to_string(range.size()); - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - if ((left_open_interval && scalar <= range[0]) || (!left_open_interval && scalar < range[0])) { - std::string interval_description = left_open_interval ? " greater than " : " greater than or equal to "; - std::string err_msg = op_name + ":" + scalar_name + " must be" + interval_description + std::to_string(range[0]) + - ", got: " + std::to_string(scalar); - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - if (range.size() == 2) { - if ((right_open_interval && scalar >= range[1]) || (!right_open_interval && scalar > range[1])) { - std::string left_bracket = left_open_interval ? "(" : "["; - std::string right_bracket = right_open_interval ? ")" : "]"; - std::string err_msg = op_name + ":" + scalar_name + " is out of range " + left_bracket + - std::to_string(range[0]) + ", " + std::to_string(range[1]) + right_bracket + - ", got: " + std::to_string(scalar); - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - } - return Status::OK(); -} - -// Helper function to validate color attribute -Status ValidateVectorColorAttribute(const std::string &op_name, const std::string &attr_name, - const std::vector &attr, const std::vector &range); - -// Helper function to validate fill value -Status ValidateVectorFillvalue(const std::string &op_name, const std::vector &fill_value); - -// Helper function to validate mean/std value -Status ValidateVectorMeanStd(const std::string &op_name, const std::vector &mean, const std::vector &std); - -// Helper function to validate padding -Status ValidateVectorPadding(const std::string &op_name, const std::vector &padding); - -// Helper function to validate positive value -Status ValidateVectorPositive(const std::string &op_name, const std::string &vec_name, const std::vector &vec); - -// Helper function to validate non-negative value -Status ValidateVectorNonNegative(const std::string &op_name, const std::string &vec_name, - const std::vector &vec); - -// Helper function to validate size of size -Status ValidateVectorSize(const std::string &op_name, const std::vector &size); - -// Helper function to validate scale -Status ValidateVectorScale(const std::string &op_name, const std::vector &scale); - -// Helper function to validate ratio -Status ValidateVectorRatio(const std::string &op_name, const std::vector &ratio); - -// Helper function to validate transforms -Status ValidateVectorTransforms(const std::string &op_name, - const std::vector> &transforms); - -// Helper function to compare float value -bool CmpFloat(const float a, const float b, float epsilon = 0.0000000001f); - // Transform operations for performing data transformation. namespace transforms { diff --git a/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt index 2c44e33c69..ee1f8ac9a4 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(image) add_subdirectory(data) +add_subdirectory(ir) file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) if(ENABLE_PYTHON) diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/kernels/ir/CMakeLists.txt new file mode 100644 index 0000000000..dc1ac5aa6c --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") +set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) + +set(DATASET_KERNELS_IR_SRC_FILES + validators.cc + ) + +add_library(kernels-ir OBJECT ${DATASET_KERNELS_IR_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/tensor_operation.h b/mindspore/ccsrc/minddata/dataset/kernels/ir/tensor_operation.h new file mode 100644 index 0000000000..37428c5a13 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/tensor_operation.h @@ -0,0 +1,60 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IR_TENSOR_OPERATION_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IR_TENSOR_OPERATION_H_ + +#include +#include + +#include "minddata/dataset/kernels/tensor_op.h" +#include "minddata/dataset/util/status.h" + +namespace mindspore { +namespace dataset { + +// Abstract class to represent a dataset in the data pipeline. +class TensorOperation : public std::enable_shared_from_this { + public: + /// \brief Constructor + TensorOperation() : random_op_(false) {} + + /// \brief Constructor + explicit TensorOperation(bool random) : random_op_(random) {} + + /// \brief Destructor + ~TensorOperation() = default; + + /// \brief Pure virtual function to convert a TensorOperation class into a runtime TensorOp object. + /// \return shared pointer to the newly created TensorOp. + virtual std::shared_ptr Build() = 0; + + virtual Status ValidateParams() = 0; + + virtual std::string Name() const = 0; + + /// \brief Check whether the operation is deterministic. + /// \return true if this op is a random op (returns non-deterministic result e.g. RandomCrop) + bool IsRandomOp() const { return random_op_; } + + virtual Status to_json(nlohmann::json *out_json) { return Status::OK(); } + + protected: + bool random_op_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IR_TENSOR_OPERATION_H_ diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc new file mode 100644 index 0000000000..8967927b9f --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc @@ -0,0 +1,195 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/kernels/ir/validators.h" + +namespace mindspore { +namespace dataset { + +/* ####################################### Validator Functions ############################################ */ +Status ValidateProbability(const std::string &op_name, const float probability) { + if (probability < 0.0 || probability > 1.0) { + std::string err_msg = op_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + return Status::OK(); +} + +Status ValidateIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, true)); + return Status::OK(); +} + +Status ValidateFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, true)); + return Status::OK(); +} + +Status ValidateVectorFillvalue(const std::string &op_name, const std::vector &fill_value) { + if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) { + std::string err_msg = + op_name + ": fill_value expecting size 1 or 3, got fill_value.size(): " + std::to_string(fill_value.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + // Note that fill_value need to be in range [0, 255], + // but we omit the check since its type is uint8_t + return Status::OK(); +} + +Status ValidateVectorColorAttribute(const std::string &op_name, const std::string &attr_name, + const std::vector &attr, const std::vector &range) { + if (attr.empty() || attr.size() > 2) { + std::string err_msg = op_name + ":" + attr_name + " expecting size 1 or 2, but got: " + std::to_string(attr.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (auto &attr_val : attr) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, attr_name, attr_val, range, false, false)); + } + if (attr.size() == 2 && (attr[0] > attr[1])) { + std::string err_msg = op_name + ":" + attr_name + + " lower bound must be less or equal to upper bound, got lb: " + std::to_string(attr[0]) + + ", ub: " + std::to_string(attr[1]); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + return Status::OK(); +} + +Status ValidateVectorMeanStd(const std::string &op_name, const std::vector &mean, + const std::vector &std) { + if (mean.size() != 3) { + std::string err_msg = op_name + ": mean expecting size 3, got size: " + std::to_string(mean.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + if (std.size() != 3) { + std::string err_msg = op_name + ": std expecting size 3, got size: " + std::to_string(std.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + // check std/mean value + for (int32_t i = 0; i < std.size(); ++i) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, "mean", mean[i], {0.0, 255.0}, false, false)); + RETURN_IF_NOT_OK(ValidateScalar(op_name, "std", std[i], {0.0, 255.0}, true, false)); + } + + return Status::OK(); +} + +Status ValidateVectorPadding(const std::string &op_name, const std::vector &padding) { + if (padding.empty() || padding.size() == 3 || padding.size() > 4) { + std::string err_msg = op_name + ": padding expecting size 1, 2 or 4, got size: " + std::to_string(padding.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (const auto &pad_val : padding) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, "padding", pad_val, {0, INT_MAX}, false, false)); + } + + return Status::OK(); +} + +Status ValidateVectorPositive(const std::string &op_name, const std::string &vec_name, + const std::vector &vec) { + for (const auto &vec_val : vec) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, vec_name, vec_val, {0}, true)); + } + + return Status::OK(); +} + +Status ValidateVectorNonNegative(const std::string &op_name, const std::string &vec_name, + const std::vector &vec) { + for (const auto &vec_val : vec) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, vec_name, vec_val, {0}, false)); + } + + return Status::OK(); +} + +Status ValidateVectorSize(const std::string &op_name, const std::vector &size) { + if (size.empty() || size.size() > 2) { + std::string err_msg = op_name + ": size expecting size 2, got size.size(): " + std::to_string(size.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (const auto &size_val : size) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, "size", size_val, {0, INT_MAX}, true, false)); + } + + return Status::OK(); +} + +Status ValidateVectorScale(const std::string &op_name, const std::vector &scale) { + if (scale.size() != 2) { + std::string err_msg = op_name + ": scale expecting size 2, got scale.size(): " + std::to_string(scale.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", scale[0], {0}, false)); + RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", scale[1], {0}, true)); + if (scale[1] < scale[0]) { + std::string err_msg = op_name + ": scale must be in the format of (min, max)."; + MS_LOG(ERROR) << op_name + ": scale must be in the format of (min, max), but got: " << scale; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + return Status::OK(); +} + +Status ValidateVectorRatio(const std::string &op_name, const std::vector &ratio) { + if (ratio.size() != 2) { + std::string err_msg = op_name + ": ratio expecting size 2, got ratio.size(): " + std::to_string(ratio.size()); + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", ratio[0], {0}, true)); + RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", ratio[1], {0}, true)); + if (ratio[1] < ratio[0]) { + std::string err_msg = op_name + ": ratio must be in the format of (min, max)."; + MS_LOG(ERROR) << op_name + ": ratio must be in the format of (min, max), but got: " << ratio; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + return Status::OK(); +} + +Status ValidateVectorTransforms(const std::string &op_name, + const std::vector> &transforms) { + if (transforms.empty()) { + std::string err_msg = op_name + ": transform list must not be empty."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + for (int32_t i = 0; i < transforms.size(); ++i) { + if (transforms[i] == nullptr) { + std::string err_msg = + op_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr."; + MS_LOG(ERROR) << err_msg; + RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + } + + return Status::OK(); +} + +bool CmpFloat(const float a, const float b, float epsilon) { return (std::fabs(a - b) < epsilon); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h new file mode 100644 index 0000000000..555a4369c7 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h @@ -0,0 +1,105 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IR_VALIDATORS_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IR_VALIDATORS_H_ + +#include +#include +#include + +#include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/util/status.h" + +namespace mindspore { +namespace dataset { + +class TensorOperation; + +// Helper function to validate probability +Status ValidateProbability(const std::string &op_name, const float probability); + +// Helper function to positive int scalar +Status ValidateIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar); + +// Helper function to positive float scalar +Status ValidateFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar); + +// Helper function to validate scalar +template +Status ValidateScalar(const std::string &op_name, const std::string &scalar_name, const T scalar, + const std::vector &range, bool left_open_interval = false, bool right_open_interval = false) { + if (range.empty() || range.size() > 2) { + std::string err_msg = "Range check expecting size 1 or 2, but got: " + std::to_string(range.size()); + return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); + } + if ((left_open_interval && scalar <= range[0]) || (!left_open_interval && scalar < range[0])) { + std::string interval_description = left_open_interval ? " greater than " : " greater than or equal to "; + std::string err_msg = op_name + ":" + scalar_name + " must be" + interval_description + std::to_string(range[0]) + + ", got: " + std::to_string(scalar); + return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); + } + if (range.size() == 2) { + if ((right_open_interval && scalar >= range[1]) || (!right_open_interval && scalar > range[1])) { + std::string left_bracket = left_open_interval ? "(" : "["; + std::string right_bracket = right_open_interval ? ")" : "]"; + std::string err_msg = op_name + ":" + scalar_name + " is out of range " + left_bracket + + std::to_string(range[0]) + ", " + std::to_string(range[1]) + right_bracket + + ", got: " + std::to_string(scalar); + return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); + } + } + return Status::OK(); +} + +// Helper function to validate color attribute +Status ValidateVectorColorAttribute(const std::string &op_name, const std::string &attr_name, + const std::vector &attr, const std::vector &range); + +// Helper function to validate fill value +Status ValidateVectorFillvalue(const std::string &op_name, const std::vector &fill_value); + +// Helper function to validate mean/std value +Status ValidateVectorMeanStd(const std::string &op_name, const std::vector &mean, const std::vector &std); + +// Helper function to validate padding +Status ValidateVectorPadding(const std::string &op_name, const std::vector &padding); + +// Helper function to validate positive value +Status ValidateVectorPositive(const std::string &op_name, const std::string &vec_name, const std::vector &vec); + +// Helper function to validate non-negative value +Status ValidateVectorNonNegative(const std::string &op_name, const std::string &vec_name, + const std::vector &vec); + +// Helper function to validate size of size +Status ValidateVectorSize(const std::string &op_name, const std::vector &size); + +// Helper function to validate scale +Status ValidateVectorScale(const std::string &op_name, const std::vector &scale); + +// Helper function to validate ratio +Status ValidateVectorRatio(const std::string &op_name, const std::vector &ratio); + +// Helper function to validate transforms +Status ValidateVectorTransforms(const std::string &op_name, + const std::vector> &transforms); + +// Helper function to compare float value +bool CmpFloat(const float a, const float b, float epsilon = 0.0000000001f); +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IR_VALIDATORS_H_ diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index 75779ca54a..a41f726dce 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -99,6 +99,7 @@ AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/util MINDDATA_UTIL_SRC_FILES) AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/image/lite_cv MINDDATA_KERNELS_IMAGE_LITE_CV_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/ir MINDDATA_KERNELS_IR_SRC_FILES) if(BUILD_MINDDATA STREQUAL "full") include_directories("${MINDDATA_DIR}/kernels/image") @@ -200,6 +201,7 @@ if(BUILD_MINDDATA STREQUAL "full") ${MINDDATA_DIR}/kernels/data/random_choice_op.cc ${MINDDATA_DIR}/kernels/data/type_cast_op.cc ${MINDDATA_DIR}/kernels/image/exif_utils.cc + ${MINDDATA_DIR}/kernels/ir/validators.cc ${MINDDATA_DIR}/callback/callback_manager.cc ${MINDDATA_DIR}/util/task_manager.cc ${MINDDATA_DIR}/util/services.cc @@ -294,6 +296,7 @@ elseif(BUILD_MINDDATA STREQUAL "wrapper") ${CORE_DIR}/utils/ms_utils.cc ${MINDDATA_TODAPI_SRC} ${MINDSPORE_LITE_CXXAPI_SRC} + ${MINDDATA_DIR}/kernels/ir/validators.cc ) find_package(Threads REQUIRED) @@ -387,6 +390,7 @@ elseif(BUILD_MINDDATA STREQUAL "lite") ${MINDDATA_DIR}/api/transforms.cc ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/log_adapter.cc ${CORE_DIR}/utils/ms_utils.cc + ${MINDDATA_DIR}/kernels/ir/validators.cc ) target_link_libraries(minddata-lite