| @@ -55,9 +55,9 @@ struct Execute::ExtraInfo { | |||
| }; | |||
| // FIXME - Temporarily overload Execute to support both TensorOperation and TensorTransform | |||
| Execute::Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType, uint32_t device_id) { | |||
| Execute::Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice device_type, uint32_t device_id) { | |||
| ops_.emplace_back(std::move(op)); | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| @@ -65,72 +65,72 @@ Execute::Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType | |||
| Status rc = device_resource_->InitResource(device_id); | |||
| if (!rc.IsOk()) { | |||
| device_resource_ = nullptr; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail"; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail."; | |||
| } | |||
| } | |||
| #endif | |||
| } | |||
| Execute::Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType, uint32_t device_id) { | |||
| Execute::Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice device_type, uint32_t device_id) { | |||
| // Initialize the op and other context | |||
| transforms_.emplace_back(op); | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| device_resource_ = std::make_shared<AscendResource>(); | |||
| Status rc = device_resource_->InitResource(device_id); | |||
| if (!rc.IsOk()) { | |||
| device_resource_ = nullptr; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail"; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail."; | |||
| } | |||
| } | |||
| #endif | |||
| } | |||
| Execute::Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType, uint32_t device_id) { | |||
| Execute::Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice device_type, uint32_t device_id) { | |||
| // Initialize the transforms_ and other context | |||
| std::shared_ptr<TensorOperation> operation = op.get().Parse(); | |||
| ops_.emplace_back(std::move(operation)); | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| info_->init_with_shared_ptr_ = false; | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| device_resource_ = std::make_shared<AscendResource>(); | |||
| Status rc = device_resource_->InitResource(device_id); | |||
| if (!rc.IsOk()) { | |||
| device_resource_ = nullptr; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail"; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail."; | |||
| } | |||
| } | |||
| #endif | |||
| } | |||
| // Execute function for the example case: auto decode(new vision::Decode()); | |||
| Execute::Execute(TensorTransform *op, MapTargetDevice deviceType, uint32_t device_id) { | |||
| Execute::Execute(TensorTransform *op, MapTargetDevice device_type, uint32_t device_id) { | |||
| // Initialize the transforms_ and other context | |||
| std::shared_ptr<TensorTransform> smart_ptr_op(op); | |||
| transforms_.emplace_back(smart_ptr_op); | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| device_resource_ = std::make_shared<AscendResource>(); | |||
| Status rc = device_resource_->InitResource(device_id); | |||
| if (!rc.IsOk()) { | |||
| device_resource_ = nullptr; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail"; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail."; | |||
| } | |||
| } | |||
| #endif | |||
| } | |||
| Execute::Execute(std::vector<std::shared_ptr<TensorOperation>> ops, MapTargetDevice deviceType, uint32_t device_id) | |||
| : ops_(std::move(ops)), device_type_(deviceType) { | |||
| Execute::Execute(std::vector<std::shared_ptr<TensorOperation>> ops, MapTargetDevice device_type, uint32_t device_id) | |||
| : ops_(std::move(ops)), device_type_(device_type) { | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| @@ -138,46 +138,46 @@ Execute::Execute(std::vector<std::shared_ptr<TensorOperation>> ops, MapTargetDev | |||
| Status rc = device_resource_->InitResource(device_id); | |||
| if (!rc.IsOk()) { | |||
| device_resource_ = nullptr; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail"; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail."; | |||
| } | |||
| } | |||
| #endif | |||
| } | |||
| Execute::Execute(std::vector<std::shared_ptr<TensorTransform>> ops, MapTargetDevice deviceType, uint32_t device_id) { | |||
| Execute::Execute(std::vector<std::shared_ptr<TensorTransform>> ops, MapTargetDevice device_type, uint32_t device_id) { | |||
| // Initialize the transforms_ and other context | |||
| transforms_ = ops; | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| device_resource_ = std::make_shared<AscendResource>(); | |||
| Status rc = device_resource_->InitResource(device_id); | |||
| if (!rc.IsOk()) { | |||
| device_resource_ = nullptr; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail"; | |||
| MS_LOG(ERROR) << "Initialize Ascend310 resource fail."; | |||
| } | |||
| } | |||
| #endif | |||
| } | |||
| Execute::Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops, MapTargetDevice deviceType, | |||
| Execute::Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops, MapTargetDevice device_type, | |||
| uint32_t device_id) { | |||
| // Initialize the transforms_ and other context | |||
| if (deviceType == MapTargetDevice::kCpu) { | |||
| if (device_type == MapTargetDevice::kCpu) { | |||
| (void)std::transform( | |||
| ops.begin(), ops.end(), std::back_inserter(ops_), | |||
| [](TensorTransform &operation) -> std::shared_ptr<TensorOperation> { return operation.Parse(); }); | |||
| } else { | |||
| for (auto &op : ops) { | |||
| ops_.emplace_back(op.get().Parse(deviceType)); | |||
| ops_.emplace_back(op.get().Parse(device_type)); | |||
| } | |||
| } | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| info_->init_with_shared_ptr_ = false; | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| device_resource_ = std::make_shared<AscendResource>(); | |||
| @@ -191,7 +191,7 @@ Execute::Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops, | |||
| } | |||
| // Execute function for the example vector case: auto decode(new vision::Decode()); | |||
| Execute::Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice deviceType, uint32_t device_id) { | |||
| Execute::Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice device_type, uint32_t device_id) { | |||
| // Initialize the transforms_ and other context | |||
| for (auto &op : ops) { | |||
| std::shared_ptr<TensorTransform> smart_ptr_op(op); | |||
| @@ -199,7 +199,7 @@ Execute::Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice devi | |||
| } | |||
| info_ = std::make_shared<ExtraInfo>(); | |||
| device_type_ = deviceType; | |||
| device_type_ = device_type; | |||
| #ifdef ENABLE_ACL | |||
| if (device_type_ == MapTargetDevice::kAscend310) { | |||
| device_resource_ = std::make_shared<AscendResource>(); | |||
| @@ -226,15 +226,15 @@ Execute::~Execute() { | |||
| Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor *output) { | |||
| // Validate input tensor | |||
| CHECK_FAIL_RETURN_UNEXPECTED(input.DataSize() > 0, "Input Tensor has no data"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(validate_device_(), "Device Type should be 'Ascend310' or 'CPU'"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(input.DataSize() > 0, "Input Tensor has no data."); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(ValidateDevice(), "Device Type should be 'Ascend310' or 'CPU'."); | |||
| // Parse TensorTransform transforms_ into TensorOperation ops_ | |||
| if (info_->init_with_shared_ptr_) { | |||
| RETURN_IF_NOT_OK(ParseTransforms_()); | |||
| RETURN_IF_NOT_OK(ParseTransforms()); | |||
| info_->init_with_shared_ptr_ = false; | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided."); | |||
| // Validate and build runtime ops | |||
| std::vector<std::shared_ptr<TensorOp>> transforms; // record the transformations | |||
| @@ -244,10 +244,11 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor | |||
| for (int32_t i = 0; i < ops_.size(); i++) { | |||
| if (ops_[i] == nullptr) { | |||
| MS_LOG(ERROR) << "Input TensorOperation[" | |||
| << std::to_string(i) + "] is unsupported on your input device:" << env_list.at(device_type_); | |||
| std::string err_msg = "Input TensorOperation[" + std::to_string(i) + | |||
| "] is unsupported on your input device:" + env_list.at(device_type_); | |||
| MS_LOG(ERROR) << err_msg; | |||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(ops_[i] != nullptr, "Input TensorOperation[" + std::to_string(i) + "] is null"); | |||
| RETURN_IF_NOT_OK(ops_[i]->ValidateParams()); | |||
| transforms.emplace_back(ops_[i]->Build()); | |||
| } | |||
| @@ -285,7 +286,7 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor | |||
| *output = mindspore::MSTensor(std::make_shared<DETensor>(de_tensor)); | |||
| } else { // Ascend310 case, where we must set Ascend resource on each operators | |||
| #ifdef ENABLE_ACL | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310."); | |||
| // Sink data from host into device | |||
| std::shared_ptr<mindspore::dataset::DeviceTensor> device_input; | |||
| RETURN_IF_NOT_OK(device_resource_->Sink(input, &device_input)); | |||
| @@ -300,7 +301,7 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor | |||
| // For next transform | |||
| device_input = std::move(device_output); | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data."); | |||
| *output = mindspore::MSTensor(std::make_shared<DETensor>(device_input, true)); | |||
| #endif | |||
| @@ -310,18 +311,18 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor | |||
| Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std::vector<MSTensor> *output_tensor_list) { | |||
| // Validate input tensor | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!input_tensor_list.empty(), "Input Tensor is not valid"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!input_tensor_list.empty(), "Input Tensor is not valid."); | |||
| for (auto &tensor : input_tensor_list) { | |||
| CHECK_FAIL_RETURN_UNEXPECTED(tensor.DataSize() > 0, "Input Tensor has no data"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(tensor.DataSize() > 0, "Input Tensor has no data."); | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(validate_device_(), "Device Type should be 'Ascend310' or 'CPU'"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(ValidateDevice(), "Device Type should be 'Ascend310' or 'CPU'."); | |||
| // Parse TensorTransform transforms_ into TensorOperation ops_ | |||
| if (info_->init_with_shared_ptr_) { | |||
| RETURN_IF_NOT_OK(ParseTransforms_()); | |||
| RETURN_IF_NOT_OK(ParseTransforms()); | |||
| info_->init_with_shared_ptr_ = false; | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!ops_.empty(), "Input TensorOperation should be provided."); | |||
| std::map<MapTargetDevice, std::string> env_list = { | |||
| {MapTargetDevice::kCpu, "kCpu"}, {MapTargetDevice::kGpu, "kGpu"}, {MapTargetDevice::kAscend310, "kAscend310"}}; | |||
| @@ -330,10 +331,11 @@ Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std:: | |||
| std::vector<std::shared_ptr<TensorOp>> transforms; | |||
| for (int32_t i = 0; i < ops_.size(); i++) { | |||
| if (ops_[i] == nullptr) { | |||
| MS_LOG(ERROR) << "Input TensorOperation[" | |||
| << std::to_string(i) + "] is unsupported on your input device:" << env_list.at(device_type_); | |||
| std::string err_msg = "Input TensorOperation[" + std::to_string(i) + | |||
| "] is unsupported on your input device:" + env_list.at(device_type_); | |||
| MS_LOG(ERROR) << err_msg; | |||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(ops_[i] != nullptr, "Input TensorOperation[" + std::to_string(i) + "] is null"); | |||
| RETURN_IF_NOT_OK(ops_[i]->ValidateParams()); | |||
| transforms.emplace_back(ops_[i]->Build()); | |||
| } | |||
| @@ -369,10 +371,10 @@ Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std:: | |||
| output_tensor_list->emplace_back(ms_tensor); | |||
| ++idx; | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor is not valid"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor is not valid."); | |||
| } else { // Case Ascend310 | |||
| #ifdef ENABLE_ACL | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310."); | |||
| for (auto &input_tensor : input_tensor_list) { | |||
| // Sink each data from host into device | |||
| std::shared_ptr<dataset::DeviceTensor> device_input; | |||
| @@ -398,7 +400,7 @@ Status Execute::operator()(const std::vector<MSTensor> &input_tensor_list, std:: | |||
| // Release the data on the device because we have copied one piece onto host | |||
| RETURN_IF_NOT_OK(device_resource_->DeviceDataRelease()); | |||
| } | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor vector is empty"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(!output_tensor_list->empty(), "Output Tensor vector is empty."); | |||
| #endif | |||
| } | |||
| return Status::OK(); | |||
| @@ -411,7 +413,7 @@ std::vector<uint32_t> AippSizeFilter(const std::vector<uint32_t> &resize_para, c | |||
| if ((resize_para.size() == 0 || resize_para.size() == 1) && crop_para.size() == 0) { | |||
| aipp_size = {0, 0}; | |||
| MS_LOG(WARNING) << "Dynamic input shape is not supported, incomplete aipp config file will be generated. Please " | |||
| "checkout your TensorTransform input, both src_image_size_h and src_image_size will be 0"; | |||
| "checkout your TensorTransform input, both src_image_size_h and src_image_size will be 0."; | |||
| return aipp_size; | |||
| } | |||
| @@ -456,7 +458,7 @@ std::vector<float> AippStdFilter(const std::vector<uint32_t> &normalize_para) { | |||
| std::transform(normalize_para.begin() + 3, normalize_para.end(), std::back_inserter(aipp_std), | |||
| [](uint32_t i) { return 10000 / static_cast<float>(i); }); | |||
| } else { // If 0 occurs in std vector | |||
| MS_LOG(WARNING) << "Detect 0 in std vector, please verify your input"; | |||
| MS_LOG(WARNING) << "Detect 0 in std vector, please verify your input."; | |||
| aipp_std = {1.0, 1.0, 1.0}; | |||
| } | |||
| } else { | |||
| @@ -514,7 +516,7 @@ std::string Execute::AippCfgGenerator() { | |||
| std::string config_location = "./aipp.cfg"; | |||
| #ifdef ENABLE_ACL | |||
| if (info_->init_with_shared_ptr_) { | |||
| ParseTransforms_(); | |||
| ParseTransforms(); | |||
| info_->init_with_shared_ptr_ = false; | |||
| } | |||
| std::vector<uint32_t> paras; // Record the parameters value of each Ascend operators | |||
| @@ -522,7 +524,7 @@ std::string Execute::AippCfgGenerator() { | |||
| // Validate operator ir | |||
| json ir_info; | |||
| if (ops_[i] == nullptr) { | |||
| MS_LOG(ERROR) << "Input TensorOperation[" + std::to_string(i) + "] is null"; | |||
| MS_LOG(ERROR) << "Input TensorOperation[" + std::to_string(i) + "] is null."; | |||
| return ""; | |||
| } | |||
| @@ -541,8 +543,8 @@ std::string Execute::AippCfgGenerator() { | |||
| outfile.open(config_location, std::ofstream::out); | |||
| if (!outfile.is_open()) { | |||
| MS_LOG(ERROR) << "Fail to open Aipp config file, please verify your system config(including authority)" | |||
| << "We will return empty string which represent the location of Aipp config file in this case"; | |||
| MS_LOG(ERROR) << "Fail to open Aipp config file, please verify your system config(including authority)." | |||
| << "We will return empty string which represent the location of Aipp config file in this case."; | |||
| std::string except = ""; | |||
| return except; | |||
| } | |||
| @@ -611,10 +613,10 @@ std::string Execute::AippCfgGenerator() { | |||
| bool IsEmptyPtr(std::shared_ptr<TensorTransform> api_ptr) { return api_ptr == nullptr; } | |||
| Status Execute::ParseTransforms_() { | |||
| Status Execute::ParseTransforms() { | |||
| auto iter = std::find_if(transforms_.begin(), transforms_.end(), IsEmptyPtr); | |||
| if (iter != transforms_.end()) { | |||
| std::string err_msg = "Your input TensorTransforms contain at least one nullptr, please check your input"; | |||
| std::string err_msg = "Your input TensorTransforms contain at least one nullptr, please check your input."; | |||
| MS_LOG(ERROR) << err_msg; | |||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||
| } | |||
| @@ -633,10 +635,10 @@ Status Execute::ParseTransforms_() { | |||
| return Status::OK(); | |||
| } | |||
| Status Execute::validate_device_() { | |||
| Status Execute::ValidateDevice() { | |||
| if (device_type_ != MapTargetDevice::kCpu && device_type_ != MapTargetDevice::kAscend310 && | |||
| device_type_ != MapTargetDevice::kGpu) { | |||
| std::string err_msg = "Your input device is not supported. (Option: CPU or GPU or Ascend310)"; | |||
| std::string err_msg = "Your input device is not supported. (Option: CPU or GPU or Ascend310)."; | |||
| MS_LOG(ERROR) << err_msg; | |||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||
| } | |||
| @@ -644,7 +646,7 @@ Status Execute::validate_device_() { | |||
| } | |||
| Status Execute::DeviceMemoryRelease() { | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310"); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(device_resource_, "Device resource is nullptr which is illegal under case Ascend310."); | |||
| Status rc = device_resource_->DeviceDataRelease(); | |||
| if (rc.IsError()) { | |||
| std::string err_msg = "Error in device data release"; | |||
| @@ -28,55 +28,56 @@ namespace dataset { | |||
| // Config operations for setting and getting the configuration. | |||
| namespace config { | |||
| /// \brief Function to set the seed to be used in any random generator. This is used to produce deterministic results. | |||
| /// \param[in] seed the default seed to use. | |||
| /// \brief A function to set the seed to be used in any random generator. This is used to produce deterministic results. | |||
| /// \param[in] seed The default seed to be used. | |||
| bool set_seed(int32_t seed); | |||
| /// \brief Function to get the seed. | |||
| /// \return the seed set in the configuration. | |||
| /// \brief A function to get the seed. | |||
| /// \return The seed set in the configuration. | |||
| uint32_t get_seed(); | |||
| /// \brief Function to set the number of rows to be prefetched. | |||
| /// \param[in] prefetch_size total number of rows to be prefetched. | |||
| /// \brief A function to set the number of rows to be prefetched. | |||
| /// \param[in] prefetch_size Total number of rows to be prefetched. | |||
| bool set_prefetch_size(int32_t prefetch_size); | |||
| /// \brief Function to get the prefetch size in number of rows. | |||
| /// \return total number of rows to be prefetched. | |||
| /// \brief A function to get the prefetch size in number of rows. | |||
| /// \return Total number of rows to be prefetched. | |||
| int32_t get_prefetch_size(); | |||
| /// \brief Function to set the default number of parallel workers. | |||
| /// \param[in] num_parallel_workers number of parallel workers to be used as a default for each operation. | |||
| /// \brief A function to set the default number of parallel workers. | |||
| /// \param[in] num_parallel_workers Number of parallel workers to be used as the default for each operation. | |||
| bool set_num_parallel_workers(int32_t num_parallel_workers); | |||
| /// \brief Function to get the default number of parallel workers. | |||
| /// \return number of parallel workers to be used as a default for each operation. | |||
| /// \brief A function to get the default number of parallel workers. | |||
| /// \return Number of parallel workers to be used as the default for each operation. | |||
| int32_t get_num_parallel_workers(); | |||
| /// \brief Function to set the default interval (in milliseconds) for monitor sampling. | |||
| /// \param[in] interval interval (in milliseconds) to be used for performance monitor sampling. | |||
| /// \brief A function to set the default interval (in milliseconds) for monitor sampling. | |||
| /// \param[in] interval Interval (in milliseconds) to be used for performance monitor sampling. | |||
| bool set_monitor_sampling_interval(int32_t interval); | |||
| /// \brief Function to get the default interval of performance monitor sampling. | |||
| /// \return interval (in milliseconds) for performance monitor sampling. | |||
| /// \brief A function to get the default interval of performance monitor sampling. | |||
| /// \return Interval (in milliseconds) for performance monitor sampling. | |||
| int32_t get_monitor_sampling_interval(); | |||
| /// \brief Function to set the default timeout (in seconds) for DSWaitedCallback. In case of a deadlock, the wait | |||
| /// \brief A function to set the default timeout (in seconds) for DSWaitedCallback. In case of a deadlock, the wait | |||
| /// function will exit after the timeout period. | |||
| /// \param[in] timeout timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock. | |||
| /// \param[in] timeout Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock. | |||
| bool set_callback_timeout(int32_t timeout); | |||
| /// \brief Function to get the default timeout for DSWaitedCallback. In case of a deadback, the wait function will exit | |||
| /// after the timeout period. | |||
| /// \return the duration in seconds. | |||
| /// \brief A function to get the default timeout for DSWaitedCallback. In case of a deadback, the wait function | |||
| /// will exit after the timeout period. | |||
| /// \return The duration in seconds. | |||
| int32_t get_callback_timeout(); | |||
| /// \brief Function to load configuration from a file. | |||
| /// \param[in] file path of the configuration file to be loaded. | |||
| /// \note This API exists because std::string will constrained by ABI compile option while char don't. | |||
| /// \brief A function to load the configuration from a file. | |||
| /// \param[in] file Path of the configuration file to be loaded. | |||
| /// \note The reason for using this API is that std::string will be constrained by the | |||
| /// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction. | |||
| bool load(const std::vector<char> &file); | |||
| /// \brief Function to load configuration from a file. | |||
| /// \param[in] file path of the configuration file to be loaded. | |||
| /// \brief A function to load the configuration from a file. | |||
| /// \param[in] file Path of the configuration file to be loaded. | |||
| inline bool load(std::string file) { return load(StringToChar(file)); } | |||
| } // namespace config | |||
| @@ -51,10 +51,10 @@ enum class ShuffleMode { | |||
| /// \brief The method of padding. | |||
| enum class BorderType { | |||
| kConstant = 0, ///< Fills the border with constant values. | |||
| kEdge = 1, ///< Fills the border with the last value on the edge. | |||
| kReflect = 2, ///< Reflects the values on the edge omitting the last value of edge. | |||
| kSymmetric = 3 ///< Reflects the values on the edge repeating the last value of edge. | |||
| kConstant = 0, ///< Fill the border with constant values. | |||
| kEdge = 1, ///< Fill the border with the last value on the edge. | |||
| kReflect = 2, ///< Reflect the values on the edge omitting the last value of edge. | |||
| kSymmetric = 3 ///< Reflect the values on the edge repeating the last value of edge. | |||
| }; | |||
| /// \brief Possible options for Image format types in a batch. | |||
| @@ -108,7 +108,7 @@ enum class SentencePieceModel { | |||
| /// \brief Possible options to specify a specific normalize mode. | |||
| enum class NormalizeForm { | |||
| kNone = 0, ///< Do nothing for input string tensor. | |||
| kNone = 0, ///< Keep the input string tensor unchanged. | |||
| kNfc, ///< Normalize with Normalization Form C. | |||
| kNfkc, ///< Normalize with Normalization Form KC. | |||
| kNfd, ///< Normalize with Normalization Form D. | |||
| @@ -95,55 +95,55 @@ class Dataset : public std::enable_shared_from_this<Dataset> { | |||
| /// \brief Destructor | |||
| ~Dataset() = default; | |||
| /// \brief Gets the dataset size | |||
| /// \brief Get the dataset size | |||
| /// \param[in] estimate This is only supported by some of the ops and it's used to speed up the process of getting | |||
| /// dataset size at the expense of accuracy. | |||
| /// \return Dataset size. If failed, return -1. | |||
| int64_t GetDatasetSize(bool estimate = false); | |||
| /// \brief Gets the output type | |||
| /// \brief Get the output type | |||
| /// \return A vector contains output DataType of dataset. If failed, return an empty vector. | |||
| std::vector<mindspore::DataType> GetOutputTypes(); | |||
| /// \brief Gets the output shape | |||
| /// \brief Get the output shape | |||
| /// \return A vector contains output TensorShape of dataset. If failed, return an empty vector. | |||
| std::vector<std::vector<int64_t>> GetOutputShapes(); | |||
| /// \brief Gets the batch size | |||
| /// \brief Get the batch size | |||
| /// \return Batch size configuration of dataset. | |||
| int64_t GetBatchSize(); | |||
| /// \brief Gets the repeat count | |||
| /// \brief Get the repeat count | |||
| /// \return Repeat count configuration of dataset. | |||
| int64_t GetRepeatCount(); | |||
| /// \brief Gets the number of classes | |||
| /// \brief Get the number of classes | |||
| /// \return Number of classes of dataset. If failed, return -1. | |||
| int64_t GetNumClasses(); | |||
| /// \brief Gets the column names | |||
| /// \brief Get the column names | |||
| /// \return A vector contains all column names of dataset. If failed, return an empty vector. | |||
| std::vector<std::string> GetColumnNames() { return VectorCharToString(GetColumnNamesCharIF()); } | |||
| /// \brief Gets the class indexing | |||
| /// \brief Get the class indexing | |||
| /// \return A map of ClassIndexing of dataset. If failed, return an empty map. | |||
| std::vector<std::pair<std::string, std::vector<int32_t>>> GetClassIndexing() { | |||
| return ClassIndexCharToString(GetClassIndexingCharIF()); | |||
| } | |||
| /// \brief Setter function for runtime number of workers. | |||
| /// \brief Function to set runtime number of workers. | |||
| /// \param[in] num_workers The number of threads in this operator. | |||
| /// \return Shared pointer to the original object. | |||
| std::shared_ptr<Dataset> SetNumWorkers(int32_t num_workers); | |||
| /// \brief Function to create an PullBasedIterator over the Dataset. | |||
| /// \brief A Function to create an PullBasedIterator over the Dataset. | |||
| /// \param[in] columns List of columns to be used to specify the order of columns. | |||
| /// \return Shared pointer to the Iterator. | |||
| std::shared_ptr<PullIterator> CreatePullBasedIterator(std::vector<std::vector<char>> columns = {}); | |||
| /// \brief Function to create an Iterator over the Dataset pipeline. | |||
| /// \param[in] columns List of columns to be used to specify the order of columns. | |||
| /// \param[in] num_epochs Number of epochs to run through the pipeline, default -1 which means infinite epochs. | |||
| /// \param[in] num_epochs Number of epochs to run through the pipeline (default=-1, which means infinite epochs). | |||
| /// An empty row is returned at the end of each epoch. | |||
| /// \return Shared pointer to the Iterator. | |||
| std::shared_ptr<Iterator> CreateIterator(std::vector<std::string> columns = {}, int32_t num_epochs = -1) { | |||
| @@ -674,7 +674,8 @@ class ZipDataset : public Dataset { | |||
| /// \brief Function to create a SchemaObj. | |||
| /// \param[in] schema_file Path of schema file. | |||
| /// \note This API exists because std::string will constrained by ABI compile option while char don't. | |||
| /// \note The reason for using this API is that std::string will be constrained by the | |||
| /// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction. | |||
| /// \return Shared pointer to the current schema. | |||
| std::shared_ptr<SchemaObj> SchemaCharIF(const std::vector<char> &schema_file); | |||
| @@ -1012,7 +1013,7 @@ inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const s | |||
| /// \param[in] annotation_file Path to the annotation json. | |||
| /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'. | |||
| /// \param[in] decode Decode the images after reading. | |||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.. | |||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||
| /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). | |||
| /// \param[in] extra_metadata Flag to add extra meta-data to row. (default=false) | |||
| /// \return Shared pointer to the CocoDataset. | |||
| @@ -1111,7 +1112,7 @@ class ImageFolderDataset : public Dataset { | |||
| /// \brief Function to create an ImageFolderDataset. | |||
| /// \note A source dataset that reads images from a tree of directories. | |||
| /// All images within one folder have the same label. | |||
| /// The generated dataset has two columns ["image", "label"] | |||
| /// The generated dataset has two columns ["image", "label"]. | |||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset. | |||
| /// \param[in] decode A flag to decode in ImageFolder. | |||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||
| @@ -1366,7 +1367,6 @@ inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file | |||
| /// ShuffleMode::kGlobal - Shuffle both the files and samples. | |||
| /// ShuffleMode::kInfile - Shuffle samples in file. | |||
| /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). | |||
| /// \param[in] cache Tensor cache to use (default=nullptr which means no cache is used). | |||
| /// \return Shared pointer to the MindDataDataset. | |||
| inline std::shared_ptr<MindDataDataset> MindData( | |||
| const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list = {}, | |||
| @@ -33,58 +33,62 @@ class DeviceResource; | |||
| class Execute { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] op TensorOperation to be applied in Eager mode, it accepts op in type of shared pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType = MapTargetDevice::kCpu, | |||
| /// \param[in] op TensorOperation to be applied in Eager mode, it accepts operation in type of shared pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice device_type = MapTargetDevice::kCpu, | |||
| uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] op TensorTransform to be applied in Eager mode, it accepts op in type of shared pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu, | |||
| /// \param[in] op TensorTransform to be applied in Eager mode, it accepts operation in type of shared pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice device_type = MapTargetDevice::kCpu, | |||
| uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] op TensorTransform to be applied in Eager mode, it accepts op in type of reference. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| explicit Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu, | |||
| /// \param[in] op TensorTransform to be applied in Eager mode, it accepts operation in type of reference. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice device_type = MapTargetDevice::kCpu, | |||
| uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] op TensorTransform to be applied in Eager mode, it accepts op in type of raw pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| explicit Execute(TensorTransform *op, MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| /// \param[in] op TensorTransform to be applied in Eager mode, it accepts operation in type of raw pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(TensorTransform *op, MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] ops A vector of TensorOperations to be applied in Eager mode, it accepts op in type of shared pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| /// \param[in] ops A vector of TensorOperations to be applied in Eager mode, it accepts operation | |||
| /// in type of shared pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(std::vector<std::shared_ptr<TensorOperation>> ops, | |||
| MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts op in type of shared pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| /// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts operation | |||
| /// in type of shared pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(std::vector<std::shared_ptr<TensorTransform>> ops, | |||
| MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts op in type of raw pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| /// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts operation | |||
| /// in type of raw pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(const std::vector<std::reference_wrapper<TensorTransform>> ops, | |||
| MapTargetDevice deviceType = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| MapTargetDevice device_type = MapTargetDevice::kCpu, uint32_t device_id = 0); | |||
| /// \brief Constructor. | |||
| /// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts op in type of raw pointer. | |||
| /// \param[in] deviceType Target device env to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device id to perform operation, only valid when deviceType=kAscend310 (default=0). | |||
| explicit Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice deviceType = MapTargetDevice::kCpu, | |||
| /// \param[in] ops A vector of TensorTransforms to be applied in Eager mode, it accepts operation | |||
| /// in type of raw pointer. | |||
| /// \param[in] device_type Target device environment to perform operation, can be kCPU/kGPU/kAscend310 (default=kCPU). | |||
| /// \param[in] device_id Target device ID to perform operation, only valid when device_type=kAscend310 (default=0). | |||
| explicit Execute(const std::vector<TensorTransform *> &ops, MapTargetDevice device_type = MapTargetDevice::kCpu, | |||
| uint32_t device_id = 0); | |||
| /// \brief Destructor. | |||
| @@ -102,23 +106,23 @@ class Execute { | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| Status operator()(const std::vector<mindspore::MSTensor> &input_tensor_list, std::vector<mindspore::MSTensor> *out); | |||
| /// \brief The function to release device memory on Ascend310. | |||
| Status DeviceMemoryRelease(); | |||
| /// \brief The function to generate AIPP configuration. | |||
| std::string AippCfgGenerator(); | |||
| private: | |||
| Status ParseTransforms_(); | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| Status ParseTransforms(); | |||
| Status validate_device_(); | |||
| /// \brief The function to validate target device setting is valid or not. | |||
| Status ValidateDevice(); | |||
| std::vector<std::shared_ptr<TensorTransform>> transforms_; | |||
| std::vector<std::shared_ptr<TensorOperation>> ops_; | |||
| MapTargetDevice device_type_; | |||
| std::shared_ptr<DeviceResource> device_resource_; | |||
| struct ExtraInfo; | |||
| std::shared_ptr<ExtraInfo> info_; | |||
| }; | |||
| @@ -47,21 +47,21 @@ using MSTensorVec = std::vector<mindspore::MSTensor>; | |||
| // Abstract class for iterating over the dataset. | |||
| class Iterator { | |||
| public: | |||
| /// \brief Constructor | |||
| /// \brief Constructor. | |||
| Iterator(); | |||
| /// \brief Destructor | |||
| /// \brief Destructor. | |||
| ~Iterator(); | |||
| /// \brief Method for building and launching the pipeline. | |||
| /// \param[in] ops - a vector of DatasetOp in the data pipeline. | |||
| /// \param[in] num_epochs Number of epochs passed down to EpochCtrlNode, default -1, infinite epochs | |||
| /// \param[in] ds The last DatasetOp in the dataset pipeline. | |||
| /// \param[in] num_epochs Number of epochs passed down to EpochCtrlNode (default=-1, which means infinite epochs). | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| Status BuildAndLaunchTree(std::shared_ptr<Dataset> ds, int32_t num_epochs); | |||
| /// \brief Function to get the next row from the data pipeline. | |||
| /// \note Type of return data is a map(with column name). | |||
| /// \param[out] row - the output tensor row. | |||
| /// \note Type of return data is a unordered_map(with column name). | |||
| /// \param[out] row The output tensor row. | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| Status GetNextRow(MSTensorMap *row) { | |||
| MSTensorMapChar row_; | |||
| @@ -72,13 +72,14 @@ class Iterator { | |||
| return s; | |||
| } | |||
| // Char interface(CharIF) of GetNextRow | |||
| // This This API exists because std::string will constrained by ABI compile option while char don't. | |||
| /// \brief Char interface(CharIF) of GetNextRow. | |||
| /// \note The reason for using this API is that std::string will be constrained by the | |||
| /// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction. | |||
| Status GetNextRowCharIF(MSTensorMapChar *row); | |||
| /// \brief Function to get the next row from the data pipeline. | |||
| /// \note Type of return data is a vector(without column name). | |||
| /// \param[out] row - the output tensor row. | |||
| /// \param[out] row The output tensor row. | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| virtual Status GetNextRow(MSTensorVec *row); | |||
| @@ -119,10 +120,10 @@ class Iterator { | |||
| class PullIterator : public Iterator { | |||
| public: | |||
| /// \brief Constructor | |||
| /// \brief Constructor. | |||
| PullIterator(); | |||
| /// \brief Destructor | |||
| /// \brief Destructor. | |||
| ~PullIterator() = default; | |||
| /// \brief Function to get next row from the data pipeline. | |||
| @@ -133,7 +134,7 @@ class PullIterator : public Iterator { | |||
| /// \brief Function to get specified rows from the data pipeline. | |||
| /// \note Type of return data is a vector(without column name). | |||
| /// \note This behavior is subject to change | |||
| /// \note This behavior is subject to change. | |||
| /// \param[in] num_rows The number of rows to fetch. | |||
| /// \param[out] row The output tensor row. | |||
| /// \return Status error code, returns OK if no error encountered else false. | |||
| @@ -141,7 +142,7 @@ class PullIterator : public Iterator { | |||
| /// \brief Method for building and launching the pipeline. | |||
| /// \note Consider making this function protected. | |||
| /// \param[in] ds - The root node that calls the function | |||
| /// \param[in] ds The root node that calls the function. | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| Status BuildAndLaunchTree(std::shared_ptr<Dataset> ds); | |||
| @@ -87,7 +87,7 @@ class DistributedSampler final : public Sampler { | |||
| ~DistributedSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| @@ -118,7 +118,7 @@ class PKSampler final : public Sampler { | |||
| ~PKSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| @@ -143,7 +143,7 @@ class RandomSampler final : public Sampler { | |||
| ~RandomSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| @@ -167,7 +167,7 @@ class SequentialSampler final : public Sampler { | |||
| ~SequentialSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| @@ -191,7 +191,7 @@ class SubsetSampler : public Sampler { | |||
| ~SubsetSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| @@ -214,7 +214,7 @@ class SubsetRandomSampler final : public SubsetSampler { | |||
| ~SubsetRandomSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| }; | |||
| @@ -236,7 +236,7 @@ class WeightedRandomSampler final : public Sampler { | |||
| ~WeightedRandomSampler() = default; | |||
| protected: | |||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||
| /// \brief The function to convert a Sampler into an IR SamplerObj. | |||
| /// \return shared pointer to the newly created SamplerObj. | |||
| std::shared_ptr<SamplerObj> Parse() const override; | |||
| @@ -40,19 +40,19 @@ namespace text { | |||
| #ifndef _WIN32 | |||
| /// \brief Tokenize a scalar tensor of UTF-8 string by specific rules. | |||
| /// \note BasicTokenizer is not supported on Windows platform yet. | |||
| /// \note BasicTokenizer is not supported on the Windows platform yet. | |||
| class BasicTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode), RegexReplace operation on input text to | |||
| /// fold the text to lower case and strip accents characters. If false, only apply | |||
| /// NormalizeUTF8('normalization_form' mode) operation on input text (default=false). | |||
| /// \param[in] keep_whitespace If true, the whitespace will be kept in out tokens (default=false). | |||
| /// \param[in] normalize_form Used to specify a specific normalize mode. This is only effective when 'lower_case' is | |||
| /// false. See NormalizeUTF8 for details (default=NormalizeForm::kNone). | |||
| /// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]', | |||
| /// '[MASK]' (default=true). | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode) and RegexReplace operations to | |||
| /// the input text to fold the text to lower case and strip accents characters. If false, only apply | |||
| /// the NormalizeUTF8('normalization_form' mode) operation to the input text (default=false). | |||
| /// \param[in] keep_whitespace If true, the whitespace will be kept in output tokens (default=false). | |||
| /// \param[in] normalize_form This parameter is used to specify a specific normalize mode. This is only effective | |||
| /// when 'lower_case' is false. See NormalizeUTF8 for details (default=NormalizeForm::kNone). | |||
| /// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and | |||
| /// '[MASK]' (default=true). | |||
| /// \param[in] with_offsets Whether to output offsets of tokens (default=false). | |||
| explicit BasicTokenizer(bool lower_case = false, bool keep_whitespace = false, | |||
| const NormalizeForm normalize_form = NormalizeForm::kNone, bool preserve_unused_token = true, | |||
| bool with_offsets = false); | |||
| @@ -61,8 +61,8 @@ class BasicTokenizer final : public TensorTransform { | |||
| ~BasicTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -70,25 +70,26 @@ class BasicTokenizer final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Tokenizer used for Bert text process. | |||
| /// \note BertTokenizer is not supported on Windows platform yet. | |||
| /// \brief A tokenizer used for Bert text process. | |||
| /// \note BertTokenizer is not supported on the Windows platform yet. | |||
| class BertTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] vocab A Vocab object. | |||
| /// \param[in] suffix_indicator Used to show that the subword is the last part of a word (default='##'). | |||
| /// \param[in] suffix_indicator This parameter is used to show that the sub-word | |||
| /// is the last part of a word (default='##'). | |||
| /// \param[in] max_bytes_per_token Tokens exceeding this length will not be further split (default=100). | |||
| /// \param[in] unknown_token When a token cannot be found, return the token directly if 'unknown_token' is an empty | |||
| /// string, else return the string specified(default='[UNK]'). | |||
| /// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode), RegexReplace operation on input text to | |||
| /// fold the text to lower case and strip accents characters. If false, only apply | |||
| /// NormalizeUTF8('normalization_form' mode) operation on input text (default=false). | |||
| /// \param[in] keep_whitespace If true, the whitespace will be kept in out tokens (default=false). | |||
| /// \param[in] normalize_form Used to specify a specific normalize mode. This is only effective when 'lower_case' is | |||
| /// false. See NormalizeUTF8 for details (default=NormalizeForm::kNone). | |||
| /// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]', | |||
| /// string, else return the specified string (default='[UNK]'). | |||
| /// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode) and RegexReplace operations to | |||
| /// the input text to fold the text to lower case and strip accents characters. If false, only apply | |||
| /// the NormalizeUTF8('normalization_form' mode) operation to the input text (default=false). | |||
| /// \param[in] keep_whitespace If true, the whitespace will be kept in output tokens (default=false). | |||
| /// \param[in] normalize_form This parameter is used to specify a specific normalize mode. This is only effective | |||
| /// when 'lower_case' is false. See NormalizeUTF8 for details (default=NormalizeForm::kNone). | |||
| /// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and | |||
| /// '[MASK]' (default=true). | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] with_offsets Whether to output offsets of tokens (default=false). | |||
| explicit BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator = "##", | |||
| int32_t max_bytes_per_token = 100, const std::string &unknown_token = "[UNK]", | |||
| bool lower_case = false, bool keep_whitespace = false, | |||
| @@ -106,8 +107,8 @@ class BertTokenizer final : public TensorTransform { | |||
| ~BertTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -115,7 +116,7 @@ class BertTokenizer final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Apply case fold operation on UTF-8 string tensor. | |||
| /// \brief Apply case fold operation on UTF-8 string tensors. | |||
| /// \return Shared pointer to the current TensorOperation. | |||
| class CaseFold final : public TensorTransform { | |||
| public: | |||
| @@ -126,26 +127,26 @@ class CaseFold final : public TensorTransform { | |||
| ~CaseFold() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| #endif | |||
| /// \brief Tokenize Chinese string into words based on dictionary. | |||
| /// \note The integrity of the HMMSEgment algorithm and MPSegment algorithm files must be confirmed. | |||
| /// \brief Tokenize a Chinese string into words based on the dictionary. | |||
| /// \note The integrity of the HMMSegment algorithm and MPSegment algorithm files must be confirmed. | |||
| class JiebaTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] hmm_path Dictionary file is used by HMMSegment algorithm. The dictionary can be obtained on the | |||
| /// official website of cppjieba. | |||
| /// \param[in] mp_path Dictionary file is used by MPSegment algorithm. The dictionary can be obtained on the | |||
| /// official website of cppjieba. | |||
| /// \param[in] mode Valid values can be any of [JiebaMode.MP, JiebaMode.HMM, JiebaMode.MIX](default=JiebaMode.MIX). | |||
| /// - JiebaMode.kMP, tokenize with MPSegment algorithm. | |||
| /// - JiebaMode.kHMM, tokenize with Hiddel Markov Model Segment algorithm. | |||
| /// - JiebaMode.kMIX, tokenize with a mix of MPSegment and HMMSegment algorithm. | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] hmm_path Dictionary file is used by the HMMSegment algorithm. The dictionary can be obtained on the | |||
| /// official website of cppjieba (https://github.com/yanyiwu/cppjieba). | |||
| /// \param[in] mp_path Dictionary file is used by the MPSegment algorithm. The dictionary can be obtained on the | |||
| /// official website of cppjieba (https://github.com/yanyiwu/cppjieba). | |||
| /// \param[in] mode Valid values can be any of JiebaMode.MP, JiebaMode.HMM and JiebaMode.MIX (default=JiebaMode.MIX). | |||
| /// - JiebaMode.kMP, tokenizes with MPSegment algorithm. | |||
| /// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm. | |||
| /// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms. | |||
| /// \param[in] with_offsets Whether to output offsets of tokens (default=false). | |||
| explicit JiebaTokenizer(const std::string &hmm_path, const std::string &mp_path, | |||
| const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false) | |||
| : JiebaTokenizer(StringToChar(hmm_path), StringToChar(mp_path), mode, with_offsets) {} | |||
| @@ -156,46 +157,46 @@ class JiebaTokenizer final : public TensorTransform { | |||
| /// \brief Destructor | |||
| ~JiebaTokenizer() = default; | |||
| /// \brief Add user defined word to JiebaTokenizer's dictionary. | |||
| /// \brief Add a user defined word to the JiebaTokenizer's dictionary. | |||
| /// \param[in] word The word to be added to the JiebaTokenizer instance. | |||
| /// The added word will not be written into the built-in dictionary on disk. | |||
| /// \param[in] freq The frequency of the word to be added. The higher the frequency, | |||
| /// the better chance the word will be tokenized (default=None, use default frequency). | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| /// \return Status error code, returns OK if no error is encountered. | |||
| Status AddWord(const std::string &word, int64_t freq = 0) { return AddWordChar(StringToChar(word), freq); } | |||
| /// \brief Add user defined dictionary of word-freq pairs to JiebaTokenizer's dictionary. | |||
| /// \param[in] user_dict Vector of word-freq pairs to be added to JiebaTokenizer's dictionary. | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| /// \brief Add a user defined dictionary of word-freq pairs to the JiebaTokenizer's dictionary. | |||
| /// \param[in] user_dict Vector of word-freq pairs to be added to the JiebaTokenizer's dictionary. | |||
| /// \return Status error code, returns OK if no error is encountered. | |||
| Status AddDict(const std::vector<std::pair<std::string, int64_t>> &user_dict) { | |||
| return AddDictChar(PairStringInt64ToPairCharInt64(user_dict)); | |||
| } | |||
| /// \brief Add user defined dictionary of word-freq pairs to JiebaTokenizer's dictionary from a file. | |||
| /// Only valid word-freq pairs in user provided file will be added into the dictionary. | |||
| /// Rows containing invalid input will be ignored, no error nor warning Status is returned. | |||
| /// \brief Add user defined dictionary of word-freq pairs to the JiebaTokenizer's dictionary from a file. | |||
| /// Only valid word-freq pairs in user defined file will be added into the dictionary. | |||
| /// Rows containing invalid inputs will be ignored, no error nor warning status is returned. | |||
| /// \param[in] file_path Path to the dictionary which includes user defined word-freq pairs. | |||
| /// \return Status error code, returns OK if no error encountered. | |||
| /// \return Status error code, returns OK if no error is encountered. | |||
| Status AddDict(const std::string &file_path) { return AddDictChar(StringToChar(file_path)); } | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| /// \brief Parser user defined word by file. | |||
| /// \brief Parser user defined words by files. | |||
| /// \param[in] file_path Path to the user defined file. | |||
| /// \param[in] user_dict Vector of word-freq pairs extracted from the user provided file. | |||
| /// \param[in] user_dict Vector of word-freq pairs extracted from the user defined file. | |||
| Status ParserFile(const std::string &file_path, std::vector<std::pair<std::string, int64_t>> *const user_dict); | |||
| /// \brief Used to translate all API string to vector of char and back | |||
| /// \brief Used to translate all API strings to vector of char and reverse. | |||
| Status AddWordChar(const std::vector<char> &word, int64_t freq = 0); | |||
| /// \brief Used to translate all API string to vector of char and back | |||
| /// \brief Used to translate all API strings to vector of char and reverse. | |||
| Status AddDictChar(const std::vector<std::pair<std::vector<char>, int64_t>> &user_dict); | |||
| /// \brief Used to translate all API string to vector of char and back | |||
| /// \brief Used to translate all API strings to vector of char and reverse. | |||
| Status AddDictChar(const std::vector<char> &file_path); | |||
| struct Data; | |||
| @@ -207,9 +208,9 @@ class Lookup final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] vocab a Vocab object. | |||
| /// \param[in] unknown_token word to use for lookup if the word being looked up is out of Vocabulary (oov). | |||
| /// If unknown_token is oov, runtime error will be thrown. If unknown_token is {}, which means that not to | |||
| /// specify unknown_token when word being out of Vocabulary (default={}). | |||
| /// \param[in] unknown_token Word is used for lookup. In case of the word is out of vocabulary (OOV), | |||
| /// the result of lookup will be replaced to unknown_token. If the unknown_token is not specified or it is OOV, | |||
| /// runtime error will be thrown (default={}, means no unknown_token is specified). | |||
| /// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool. | |||
| /// (default=mindspore::DataType::kNumberTypeInt32). | |||
| explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::string> &unknown_token = {}, | |||
| @@ -223,8 +224,8 @@ class Lookup final : public TensorTransform { | |||
| ~Lookup() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -232,17 +233,17 @@ class Lookup final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief TensorOp to generate n-gram from a 1-D string Tensor. | |||
| /// \brief Generate n-gram from a 1-D string Tensor. | |||
| class Ngram final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] ngrams ngrams is a vector of positive integers. For example, if ngrams={4, 3}, then the result | |||
| /// would be a 4-gram followed by a 3-gram in the same tensor. If the number of words is not enough to make up | |||
| /// for a n-gram, an empty string will be returned. | |||
| /// would be a 4-gram followed by a 3-gram in the same tensor. If the number of words is not enough to make up | |||
| /// a n-gram, an empty string will be returned. | |||
| /// \param[in] left_pad {"pad_token", pad_width}. Padding performed on left side of the sequence. pad_width will | |||
| /// be capped at n-1. left_pad=("_",2) would pad left side of the sequence with "__" (default={"", 0}}). | |||
| /// be capped at n-1. left_pad=("_",2) would pad the left side of the sequence with "__" (default={"", 0}}). | |||
| /// \param[in] right_pad {"pad_token", pad_width}. Padding performed on right side of the sequence.pad_width will | |||
| /// be capped at n-1. right_pad=("-":2) would pad right side of the sequence with "--" (default={"", 0}}). | |||
| /// be capped at n-1. right_pad=("-":2) would pad the right side of the sequence with "--" (default={"", 0}}). | |||
| /// \param[in] separator Symbol used to join strings together (default=" "). | |||
| explicit Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::string, int32_t> &left_pad = {"", 0}, | |||
| const std::pair<std::string, int32_t> &right_pad = {"", 0}, const std::string &separator = " ") | |||
| @@ -255,8 +256,8 @@ class Ngram final : public TensorTransform { | |||
| ~Ngram() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -265,7 +266,7 @@ class Ngram final : public TensorTransform { | |||
| }; | |||
| #ifndef _WIN32 | |||
| /// \brief Apply normalize operation on UTF-8 string tensor. | |||
| /// \brief Apply normalize operation to UTF-8 string tensors. | |||
| class NormalizeUTF8 final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -273,19 +274,19 @@ class NormalizeUTF8 final : public TensorTransform { | |||
| /// NormalizeForm::kNfkc, | |||
| /// NormalizeForm::kNfd, NormalizeForm::kNfkd](default=NormalizeForm::kNfkc). | |||
| /// See http://unicode.org/reports/tr15/ for details. | |||
| /// - NormalizeForm.NONE, do nothing for input string tensor. | |||
| /// - NormalizeForm.NFC, normalize with Normalization Form C. | |||
| /// - NormalizeForm.NFKC, normalize with Normalization Form KC. | |||
| /// - NormalizeForm.NFD, normalize with Normalization Form D. | |||
| /// - NormalizeForm.NFKD, normalize with Normalization Form KD. | |||
| /// - NormalizeForm.NONE, remain the input string tensor unchanged. | |||
| /// - NormalizeForm.NFC, normalizes with Normalization Form C. | |||
| /// - NormalizeForm.NFKC, normalizes with Normalization Form KC. | |||
| /// - NormalizeForm.NFD, normalizes with Normalization Form D. | |||
| /// - NormalizeForm.NFKD, normalizes with Normalization Form KD. | |||
| explicit NormalizeUTF8(NormalizeForm normalize_form = NormalizeForm::kNfkc); | |||
| /// \brief Destructor | |||
| ~NormalizeUTF8() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -293,13 +294,13 @@ class NormalizeUTF8 final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Replace UTF-8 string tensor with 'replace' according to regular expression 'pattern'. | |||
| /// \brief Replace a UTF-8 string tensor with 'replace' according to regular expression 'pattern'. | |||
| class RegexReplace final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] pattern The regex expression patterns. | |||
| /// \param[in] replace The string to replace matched element. | |||
| /// \param[in] replace_all Confirm whether to replace all. If false, only replace first matched element; | |||
| /// \param[in] replace The string to replace the matched element. | |||
| /// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element; | |||
| /// if true, replace all matched elements (default=true). | |||
| explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true) | |||
| : RegexReplace(StringToChar(pattern), StringToChar(replace), replace_all) {} | |||
| @@ -310,8 +311,8 @@ class RegexReplace final : public TensorTransform { | |||
| ~RegexReplace() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -319,15 +320,15 @@ class RegexReplace final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Tokenize a scalar tensor of UTF-8 string by regex expression pattern. | |||
| /// \brief Tokenize a scalar tensor of UTF-8 string by the regex expression pattern. | |||
| class RegexTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] delim_pattern The pattern of regex delimiters. | |||
| /// \param[in] keep_delim_pattern The string matched by 'delim_pattern' can be kept as a token if it can be | |||
| /// \param[in] keep_delim_pattern The string matched with 'delim_pattern' can be kept as a token if it can be | |||
| /// matched by 'keep_delim_pattern'. The default value is an empty string (""). | |||
| /// which means that delimiters will not be kept as an output token (default=""). | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] with_offsets Whether to output offsets of tokens (default=false). | |||
| explicit RegexTokenizer(std::string delim_pattern, std::string keep_delim_pattern = "", bool with_offsets = false) | |||
| : RegexTokenizer(StringToChar(delim_pattern), StringToChar(keep_delim_pattern), with_offsets) {} | |||
| @@ -338,8 +339,8 @@ class RegexTokenizer final : public TensorTransform { | |||
| ~RegexTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -348,18 +349,18 @@ class RegexTokenizer final : public TensorTransform { | |||
| }; | |||
| #endif | |||
| /// \brief Tokenize scalar token or 1-D tokens to tokens by sentencepiece. | |||
| /// \brief Tokenize a scalar token or a 1-D token to tokens by sentencepiece. | |||
| class SentencePieceTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] vocab a SentencePieceVocab object. | |||
| /// \param[in] out_type The type of output. | |||
| /// \param[in] out_type The type of the output. | |||
| SentencePieceTokenizer(const std::shared_ptr<SentencePieceVocab> &vocab, | |||
| mindspore::dataset::SPieceTokenizerOutType out_type); | |||
| /// \brief Constructor. | |||
| /// \param[in] vocab_path vocab model file path. | |||
| /// \param[in] out_type The type of output. | |||
| /// \param[in] out_type The type of the output. | |||
| SentencePieceTokenizer(const std::string &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type) | |||
| : SentencePieceTokenizer(StringToChar(vocab_path), out_type) {} | |||
| @@ -369,8 +370,8 @@ class SentencePieceTokenizer final : public TensorTransform { | |||
| ~SentencePieceTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -378,22 +379,22 @@ class SentencePieceTokenizer final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief TensorOp to construct a tensor from data (only 1-D for now), where each element in the dimension | |||
| /// \brief Construct a tensor from data (only 1-D for now), where each element in the dimension | |||
| /// axis is a slice of data starting at the corresponding position, with a specified width. | |||
| class SlidingWindow final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] width The width of the window. It must be an integer and greater than zero. | |||
| /// \param[in] axis The axis along which the sliding window is computed (default=0), axis support 0 or -1 only | |||
| /// for now. | |||
| /// \param[in] axis The axis where the sliding window is computed (default=0), axis only | |||
| /// supports 0 or -1 for now. | |||
| explicit SlidingWindow(const int32_t width, const int32_t axis = 0); | |||
| /// \brief Destructor | |||
| ~SlidingWindow() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -401,7 +402,7 @@ class SlidingWindow final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Tensor operation to convert every element of a string tensor to a number. | |||
| /// \brief Convert every element in a string tensor to a number. | |||
| /// Strings are cast according to the rules specified in the following links: | |||
| /// https://en.cppreference.com/w/cpp/string/basic_string/stof, | |||
| /// https://en.cppreference.com/w/cpp/string/basic_string/stoul, | |||
| @@ -416,8 +417,8 @@ class ToNumber final : public TensorTransform { | |||
| ~ToNumber() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -436,8 +437,8 @@ class TruncateSequencePair final : public TensorTransform { | |||
| ~TruncateSequencePair() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -449,15 +450,15 @@ class TruncateSequencePair final : public TensorTransform { | |||
| class UnicodeCharTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] with_offsets whether to output offsets of tokens (default=false). | |||
| explicit UnicodeCharTokenizer(bool with_offsets = false); | |||
| /// \brief Destructor | |||
| ~UnicodeCharTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -465,16 +466,17 @@ class UnicodeCharTokenizer final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Tokenize scalar token or 1-D tokens to 1-D subword tokens. | |||
| /// \brief Tokenize scalar token or 1-D tokens to 1-D sub-word tokens. | |||
| class WordpieceTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] vocab A Vocab object. | |||
| /// \param[in] suffix_indicator Used to show that the subword is the last part of a word (default='##'). | |||
| /// \param[in] suffix_indicator This parameter is used to show that the sub-word | |||
| /// is the last part of a word (default='##'). | |||
| /// \param[in] max_bytes_per_token Tokens exceeding this length will not be further split (default=100). | |||
| /// \param[in] unknown_token When a token cannot be found, return the token directly if 'unknown_token' is an empty | |||
| /// string, else return the string specified (default='[UNK]'). | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// string, else return the specified string (default='[UNK]'). | |||
| /// \param[in] with_offsets whether to output offsets of tokens (default=false). | |||
| explicit WordpieceTokenizer(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator = "##", | |||
| int32_t max_bytes_per_token = 100, const std::string &unknown_token = "[UNK]", | |||
| bool with_offsets = false) | |||
| @@ -488,8 +490,8 @@ class WordpieceTokenizer final : public TensorTransform { | |||
| ~WordpieceTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -502,16 +504,16 @@ class WordpieceTokenizer final : public TensorTransform { | |||
| class UnicodeScriptTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] keep_whitespace Whether or not emit whitespace tokens (default=false). | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] keep_whitespace whether to emit whitespace tokens (default=false). | |||
| /// \param[in] with_offsets whether to output offsets of tokens (default=false). | |||
| explicit UnicodeScriptTokenizer(bool keep_whitespace = false, bool with_offsets = false); | |||
| /// \brief Destructor | |||
| ~UnicodeScriptTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -523,15 +525,15 @@ class UnicodeScriptTokenizer final : public TensorTransform { | |||
| class WhitespaceTokenizer final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] with_offsets Whether or not output offsets of tokens (default=false). | |||
| /// \param[in] with_offsets whether to output offsets of tokens (default=false). | |||
| explicit WhitespaceTokenizer(bool with_offsets = false); | |||
| /// \brief Destructor | |||
| ~WhitespaceTokenizer() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to the TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| private: | |||
| @@ -103,7 +103,7 @@ class Slice { | |||
| dsize_t step_; | |||
| }; | |||
| /// \brief SliceOption used in Slice Op. | |||
| /// \brief SliceOption used in Slice TensorTransform. | |||
| class SliceOption { | |||
| public: | |||
| /// \param[in] all Slice the whole dimension | |||
| @@ -127,8 +127,7 @@ class SliceOption { | |||
| // Transform operations for performing data transformation. | |||
| namespace transforms { | |||
| /// \brief Compose Op. | |||
| /// \note Compose a list of transforms into a single transform. | |||
| /// \brief Compose a list of transforms into a single transform. | |||
| class Compose final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -145,7 +144,7 @@ class Compose final : public TensorTransform { | |||
| ~Compose() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -154,21 +153,20 @@ class Compose final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Concatenate Op. | |||
| /// \note Tensor operation that concatenates all columns into a single tensor. | |||
| /// \brief Concatenate all tensors into a single tensor. | |||
| class Concatenate final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] axis Concatenate the tensors along given axis, only support 0 or -1 so far (default=0). | |||
| /// \param[in] prepend MSTensor to be prepended to the already concatenated tensors (default={}). | |||
| /// \param[in] append MSTensor to be appended to the already concatenated tensors (default={}). | |||
| /// \param[in] prepend MSTensor to be prepended to the concatenated tensors (default={}). | |||
| /// \param[in] append MSTensor to be appended to the concatenated tensors (default={}). | |||
| explicit Concatenate(int8_t axis = 0, MSTensor prepend = {}, MSTensor append = {}); | |||
| /// \brief Destructor | |||
| ~Concatenate() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -177,8 +175,7 @@ class Concatenate final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Duplicate Op. | |||
| /// \note Duplicate the input tensor to a new output tensor. | |||
| /// \brief Duplicate the input tensor to a new output tensor. | |||
| /// The input tensor is carried over to the output list. | |||
| class Duplicate final : public TensorTransform { | |||
| public: | |||
| @@ -189,19 +186,18 @@ class Duplicate final : public TensorTransform { | |||
| ~Duplicate() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief Fill Op. | |||
| /// \note Tensor operation to fill all elements in the tensor with the specified value. | |||
| /// \brief Fill all elements in the tensor with the specified value. | |||
| /// The output tensor will have the same shape and type as the input tensor. | |||
| class Fill final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] fill_value Scalar value to fill the tensor with. | |||
| /// Can only be MSTensor of the following types from mindspore::DataType: | |||
| /// It can only be MSTensor of the following types from mindspore::DataType: | |||
| /// String, Bool, Int8/16/32/64, UInt8/16/32/64, Float16/32/64. | |||
| explicit Fill(MSTensor fill_value); | |||
| @@ -209,7 +205,7 @@ class Fill final : public TensorTransform { | |||
| ~Fill() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -218,16 +214,15 @@ class Fill final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Mask Op. | |||
| /// \note Mask content of the input tensor with the given predicate. | |||
| /// \brief Mask content of the input tensor with the given predicate. | |||
| /// Any element of the tensor that matches the predicate will be evaluated to True, otherwise False. | |||
| class Mask final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] op One of the relational operators EQ, NE LT, GT, LE or GE. | |||
| /// \param[in] constant Constant to be compared to. | |||
| /// Can only be MSTensor of str, int, float, bool. | |||
| /// \param[in] de_type Type of the generated mask. Can only be numeric or boolean datatype. | |||
| /// \param[in] op One of the relational operators: EQ, NE LT, GT, LE or GE. | |||
| /// \param[in] constant Constant to be compared to. It can only be MSTensor of the following types | |||
| /// from mindspore::DataType: String, Int, Float, Bool. | |||
| /// \param[in] de_type Type of the generated mask. It can only be numeric or boolean datatype. | |||
| /// (default=mindspore::DataType::kNumberTypeBool) | |||
| explicit Mask(RelationalOp op, MSTensor constant, | |||
| mindspore::DataType ms_type = mindspore::DataType(mindspore::DataType::kNumberTypeBool)); | |||
| @@ -236,7 +231,7 @@ class Mask final : public TensorTransform { | |||
| ~Mask() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -245,8 +240,7 @@ class Mask final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief OneHot Op. | |||
| /// \note Convert the labels into OneHot format. | |||
| /// \brief Convert the labels into OneHot format. | |||
| class OneHot final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -257,7 +251,7 @@ class OneHot final : public TensorTransform { | |||
| ~OneHot() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -266,13 +260,12 @@ class OneHot final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief PadEnd Op. | |||
| /// \note Pad input tensor according to pad_shape, need to have same rank. | |||
| /// \brief Pad input tensor according to pad_shape | |||
| class PadEnd final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] pad_shape List of integers representing the shape needed. | |||
| /// Dimensions that set to `None` will not be padded (i.e., original dim will be used). | |||
| /// \param[in] pad_shape List of integers representing the shape needed, need to have same rank with input tensor. | |||
| /// Dimensions that set to `-1` will not be padded (i.e., original dim will be used). | |||
| /// Shorter dimensions will truncate the values. | |||
| /// \param[in] pad_value Value used to pad (default={}). | |||
| explicit PadEnd(const std::vector<dsize_t> &pad_shape, MSTensor pad_value = {}); | |||
| @@ -281,7 +274,7 @@ class PadEnd final : public TensorTransform { | |||
| ~PadEnd() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -290,8 +283,7 @@ class PadEnd final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomApply Op. | |||
| /// \note Randomly perform a series of transforms with a given probability. | |||
| /// \brief Randomly perform a series of transforms with a given probability. | |||
| class RandomApply final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -311,7 +303,7 @@ class RandomApply final : public TensorTransform { | |||
| ~RandomApply() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -320,8 +312,7 @@ class RandomApply final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomChoice Op. | |||
| /// \note Randomly selects one transform from a list of transforms to perform operation. | |||
| /// \brief Randomly select one transform from a list of transforms to perform on the input tensor. | |||
| class RandomChoice final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -338,7 +329,7 @@ class RandomChoice final : public TensorTransform { | |||
| ~RandomChoice() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -347,9 +338,8 @@ class RandomChoice final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Slice Op. | |||
| /// \note Slice operation to extract a tensor out using the given n slices. | |||
| /// The functionality of Slice is similar to NumPy's indexing feature. | |||
| /// \brief Extract a tensor out using the given n slices. | |||
| /// The functionality of Slice is similar to the feature of indexing of NumPy. | |||
| /// (Currently only rank-1 tensors are supported). | |||
| class Slice final : public TensorTransform { | |||
| public: | |||
| @@ -361,7 +351,7 @@ class Slice final : public TensorTransform { | |||
| ~Slice() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -370,8 +360,7 @@ class Slice final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief TypeCast Op. | |||
| /// \note Tensor operation to cast to a given MindSpore data type. | |||
| /// \brief Cast the MindSpore data type of a tensor to another. | |||
| class TypeCast final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -382,7 +371,7 @@ class TypeCast final : public TensorTransform { | |||
| ~TypeCast() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -391,9 +380,8 @@ class TypeCast final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Unique Op. | |||
| /// \note Return an output tensor containing all the unique elements of the input tensor in | |||
| /// the same order that they occur in the input tensor. | |||
| /// \brief Return an output tensor that contains all the unique elements of the input tensor in | |||
| /// the same order as they appear in the input tensor. | |||
| class Unique final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -403,7 +391,7 @@ class Unique final : public TensorTransform { | |||
| ~Unique() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| @@ -36,12 +36,11 @@ class TensorOperation; | |||
| // Transform operations for performing computer vision. | |||
| namespace vision { | |||
| /// \brief AutoContrast TensorTransform. | |||
| /// \note Apply automatic contrast on input image. | |||
| /// \brief Apply automatic contrast on the input image. | |||
| class AutoContrast final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 100. | |||
| /// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 50. | |||
| /// \param[in] ignore Pixel values to ignore. | |||
| explicit AutoContrast(float cutoff = 0.0, std::vector<uint32_t> ignore = {}); | |||
| @@ -49,7 +48,7 @@ class AutoContrast final : public TensorTransform { | |||
| ~AutoContrast() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -58,30 +57,29 @@ class AutoContrast final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief BoundingBoxAugment TensorTransform. | |||
| /// \note Apply a given image transform on a random selection of bounding box regions of a given image. | |||
| /// \brief Apply a given image transform on a random selection of bounding box regions of a given image. | |||
| class BoundingBoxAugment final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] transform Raw pointer to a TensorTransform operation. | |||
| /// \param[in] transform Raw pointer to the TensorTransform operation. | |||
| /// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3). | |||
| explicit BoundingBoxAugment(TensorTransform *transform, float ratio = 0.3); | |||
| /// \brief Constructor. | |||
| /// \param[in] transform Smart pointer to a TensorTransform operation. | |||
| /// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3). | |||
| /// \param[in] transform Smart pointer to the TensorTransform operation. | |||
| /// \param[in] ratio Ratio of bounding boxes where augmentation is applied to. Range: [0, 1] (default=0.3). | |||
| explicit BoundingBoxAugment(const std::shared_ptr<TensorTransform> &transform, float ratio = 0.3); | |||
| /// \brief Constructor. | |||
| /// \param[in] transform Object pointer to a TensorTransform operation. | |||
| /// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3). | |||
| /// \param[in] transform Object pointer to the TensorTransform operation. | |||
| /// \param[in] ratio Ratio of bounding boxes where augmentation is applied to. Range: [0, 1] (default=0.3). | |||
| explicit BoundingBoxAugment(const std::reference_wrapper<TensorTransform> transform, float ratio = 0.3); | |||
| /// \brief Destructor. | |||
| ~BoundingBoxAugment() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -90,8 +88,7 @@ class BoundingBoxAugment final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Constructor to apply CutMix on a batch of images. | |||
| /// \note Masks a random section of each image with the corresponding part of another randomly | |||
| /// \brief Mask a random section of each image with the corresponding part of another randomly | |||
| /// selected image in that batch. | |||
| class CutMixBatch final : public TensorTransform { | |||
| public: | |||
| @@ -105,7 +102,7 @@ class CutMixBatch final : public TensorTransform { | |||
| ~CutMixBatch() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -114,8 +111,7 @@ class CutMixBatch final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief CutOut TensorOp. | |||
| /// \note Randomly cut (mask) out a given number of square patches from the input image. | |||
| /// \brief Randomly cut (mask) out a given number of square patches from the input image. | |||
| class CutOut final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -127,7 +123,7 @@ class CutOut final : public TensorTransform { | |||
| ~CutOut() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -136,8 +132,7 @@ class CutOut final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Equalize TensorTransform. | |||
| /// \note Apply histogram equalization on input image. | |||
| /// \brief Apply histogram equalization on the input image. | |||
| class Equalize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -147,13 +142,12 @@ class Equalize final : public TensorTransform { | |||
| ~Equalize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief HorizontalFlip TensorTransform. | |||
| /// \note Flip the input image horizontally. | |||
| /// \brief Flip the input image horizontally. | |||
| class HorizontalFlip final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -163,13 +157,12 @@ class HorizontalFlip final : public TensorTransform { | |||
| ~HorizontalFlip() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief HwcToChw TensorTransform. | |||
| /// \note Transpose the input image; shape (H, W, C) to shape (C, H, W). | |||
| /// \brief Transpose the input image; shape (H, W, C) to shape (C, H, W). | |||
| class HWC2CHW final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -179,13 +172,12 @@ class HWC2CHW final : public TensorTransform { | |||
| ~HWC2CHW() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief Invert TensorTransform. | |||
| /// \note Apply invert on input image in RGB mode. | |||
| /// \brief Apply invert on the input image in RGB mode. | |||
| class Invert final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -195,13 +187,12 @@ class Invert final : public TensorTransform { | |||
| ~Invert() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief MixUpBatch TensorTransform. | |||
| /// \note Apply MixUp transformation on an input batch of images and labels. The labels must be in | |||
| /// \brief Apply MixUp transformation on an input batch of images and labels. The labels must be in | |||
| /// one-hot format and Batch must be called before calling this function. | |||
| class MixUpBatch final : public TensorTransform { | |||
| public: | |||
| @@ -213,7 +204,7 @@ class MixUpBatch final : public TensorTransform { | |||
| ~MixUpBatch() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -222,15 +213,14 @@ class MixUpBatch final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief NormalizePad TensorTransform. | |||
| /// \note Normalize the input image with respect to mean and standard deviation and pad an extra | |||
| /// \brief Normalize the input image with respect to mean and standard deviation and pads an extra | |||
| /// channel with value zero. | |||
| class NormalizePad final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] mean A vector of mean values for each channel, w.r.t channel order. | |||
| /// \param[in] mean A vector of mean values for each channel, with respect to channel order. | |||
| /// The mean values must be in range [0.0, 255.0]. | |||
| /// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order. | |||
| /// \param[in] std A vector of standard deviations for each channel, with respect to channel order. | |||
| /// The standard deviation values must be in range (0.0, 255.0]. | |||
| /// \param[in] dtype The output datatype of Tensor. | |||
| /// The standard deviation values must be "float32" or "float16"(default = "float32"). | |||
| @@ -244,7 +234,7 @@ class NormalizePad final : public TensorTransform { | |||
| ~NormalizePad() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -253,16 +243,15 @@ class NormalizePad final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Pad TensorOp. | |||
| /// \note Pads the image according to padding parameters. | |||
| /// \brief Pad the image according to padding parameters. | |||
| class Pad final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] padding A vector representing the number of pixels to pad the image | |||
| /// If vector has one value, it pads all sides of the image with that value. | |||
| /// If vector has two values, it pads left and top with the first and | |||
| /// \param[in] padding A vector representing the number of pixels to pad the image. | |||
| /// If the vector has one value, it pads all sides of the image with that value. | |||
| /// If the vector has two values, it pads left and top with the first and | |||
| /// right and bottom with the second value. | |||
| /// If vector has four values, it pads left, top, right, and bottom with | |||
| /// If the vector has four values, it pads left, top, right, and bottom with | |||
| /// those values respectively. | |||
| /// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is | |||
| /// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. If 3 values are provided, | |||
| @@ -281,7 +270,7 @@ class Pad final : public TensorTransform { | |||
| ~Pad() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -290,21 +279,21 @@ class Pad final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Blends an image with its grayscale version with random weights | |||
| /// \brief Blend an image with its grayscale version with random weights | |||
| /// t and 1 - t generated from a given range. If the range is trivial | |||
| /// then the weights are determinate and t equals the bound of the interval. | |||
| /// then the weights are determinate and t equals to the bound of the interval. | |||
| class RandomColor final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] t_lb Lower bound on the range of random weights. | |||
| /// \param[in] t_lb Upper bound on the range of random weights. | |||
| /// \param[in] t_lb Lower bound random weights. | |||
| /// \param[in] t_lb Upper bound random weights. | |||
| explicit RandomColor(float t_lb, float t_ub); | |||
| /// \brief Destructor. | |||
| ~RandomColor() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -313,19 +302,18 @@ class RandomColor final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomColorAdjust TensorTransform. | |||
| /// \brief Randomly adjust the brightness, contrast, saturation, and hue of the input image. | |||
| class RandomColorAdjust final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] brightness Brightness adjustment factor. Must be a vector of one or two values | |||
| /// if it's a vector of two values it needs to be in the form of [min, max] (Default={1, 1}). | |||
| /// if it is a vector of two values it needs to be in the form of [min, max] (Default={1, 1}). | |||
| /// \param[in] contrast Contrast adjustment factor. Must be a vector of one or two values | |||
| /// if it's a vector of two values it needs to be in the form of [min, max] (Default={1, 1}). | |||
| /// if it is a vector of two values, it needs to be in the form of [min, max] (Default={1, 1}). | |||
| /// \param[in] saturation Saturation adjustment factor. Must be a vector of one or two values | |||
| /// if it's a vector of two values it needs to be in the form of [min, max] (Default={1, 1}). | |||
| /// if it is a vector of two values, it needs to be in the form of [min, max] (Default={1, 1}). | |||
| /// \param[in] hue Brightness adjustment factor. Must be a vector of one or two values | |||
| /// if it's a vector of two values it must be in the form of [min, max] where -0.5 <= min <= max <= 0.5 | |||
| /// if it is a vector of two values, it must be in the form of [min, max] where -0.5 <= min <= max <= 0.5 | |||
| /// (Default={0, 0}). | |||
| explicit RandomColorAdjust(std::vector<float> brightness = {1.0, 1.0}, std::vector<float> contrast = {1.0, 1.0}, | |||
| std::vector<float> saturation = {1.0, 1.0}, std::vector<float> hue = {0.0, 0.0}); | |||
| @@ -334,7 +322,7 @@ class RandomColorAdjust final : public TensorTransform { | |||
| ~RandomColorAdjust() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -343,22 +331,21 @@ class RandomColorAdjust final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomCrop TensorTransform. | |||
| /// \note Crop the input image at a random location. | |||
| /// \brief Crop the input image at a random location. | |||
| class RandomCrop final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the cropped image. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// \param[in] padding A vector representing the number of pixels to pad the image | |||
| /// If vector has one value, it pads all sides of the image with that value. | |||
| /// If vector has two values, it pads left and top with the first and | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| /// \param[in] padding A vector representing the number of pixels to pad the image. | |||
| /// If the vector has one value, it pads all sides of the image with that value. | |||
| /// If the vector has two values, it pads left and top with the first and | |||
| /// right and bottom with the second value. | |||
| /// If vector has four values, it pads left, top, right, and bottom with | |||
| /// If the vector has four values, it pads left, top, right, and bottom with | |||
| /// those values respectively. | |||
| /// \param[in] pad_if_needed A boolean whether to pad the image if either side is smaller than | |||
| /// the given output size. | |||
| /// \param[in] pad_if_needed A boolean indicating that whether to pad the image | |||
| /// if either side is smaller than the given output size. | |||
| /// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is | |||
| /// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. | |||
| /// If 3 values are provided, it is used to fill R, G, B channels respectively. | |||
| @@ -370,7 +357,7 @@ class RandomCrop final : public TensorTransform { | |||
| ~RandomCrop() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -379,14 +366,13 @@ class RandomCrop final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomCropDecodeResize TensorTransform. | |||
| /// \note Equivalent to RandomResizedCrop, but crops before decodes. | |||
| /// \brief Equivalent to RandomResizedCrop TensorTransform, but crop the image before decoding. | |||
| class RandomCropDecodeResize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the cropped image. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| /// \param[in] scale Range [min, max) of respective size of the | |||
| /// original size to be cropped (default=(0.08, 1.0)). | |||
| /// \param[in] ratio Range [min, max) of aspect ratio to be | |||
| @@ -403,7 +389,7 @@ class RandomCropDecodeResize final : public TensorTransform { | |||
| ~RandomCropDecodeResize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -412,23 +398,22 @@ class RandomCropDecodeResize final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomCropWithBBox TensorTransform. | |||
| /// \note Crop the input image at a random location and adjust bounding boxes accordingly. | |||
| /// If cropped area is out of bbox, the return bbox will be empty. | |||
| /// \brief Crop the input image at a random location and adjust bounding boxes accordingly. | |||
| /// If the cropped area is out of bbox, the returned bbox will be empty. | |||
| class RandomCropWithBBox final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the cropped image. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| /// \param[in] padding A vector representing the number of pixels to pad the image | |||
| /// If vector has one value, it pads all sides of the image with that value. | |||
| /// If vector has two values, it pads left and top with the first and | |||
| /// If the vector has one value, it pads all sides of the image with that value. | |||
| /// If the vector has two values, it pads left and top with the first and | |||
| /// right and bottom with the second value. | |||
| /// If vector has four values, it pads left, top, right, and bottom with | |||
| /// If the vector has four values, it pads left, top, right, and bottom with | |||
| /// those values respectively. | |||
| /// \param[in] pad_if_needed A boolean whether to pad the image if either side is smaller than | |||
| /// the given output size. | |||
| /// \param[in] pad_if_needed A boolean indicating that whether to pad the image | |||
| /// if either side is smaller than the given output size. | |||
| /// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is | |||
| /// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. | |||
| /// If 3 values are provided, it is used to fill R, G, B channels respectively. | |||
| @@ -442,7 +427,7 @@ class RandomCropWithBBox final : public TensorTransform { | |||
| ~RandomCropWithBBox() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -451,8 +436,7 @@ class RandomCropWithBBox final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomHorizontalFlip TensorTransform. | |||
| /// \note Tensor operation to perform random horizontal flip. | |||
| /// \brief Randomly flip the input image horizontally with a given probability. | |||
| class RandomHorizontalFlip final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -463,7 +447,7 @@ class RandomHorizontalFlip final : public TensorTransform { | |||
| ~RandomHorizontalFlip() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -472,8 +456,7 @@ class RandomHorizontalFlip final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomHorizontalFlipWithBBox TensorTransform. | |||
| /// \note Flip the input image horizontally, randomly with a given probability and adjust bounding boxes accordingly. | |||
| /// \brief Randomly flip the input image horizontally with a given probability and adjust bounding boxes accordingly. | |||
| class RandomHorizontalFlipWithBBox final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -484,7 +467,7 @@ class RandomHorizontalFlipWithBBox final : public TensorTransform { | |||
| ~RandomHorizontalFlipWithBBox() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -493,8 +476,7 @@ class RandomHorizontalFlipWithBBox final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomPosterize TensorTransform. | |||
| /// \note Tensor operation to perform random posterize. | |||
| /// \brief Reduce the number of bits for each color channel randomly. | |||
| class RandomPosterize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -505,7 +487,7 @@ class RandomPosterize final : public TensorTransform { | |||
| ~RandomPosterize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -514,21 +496,20 @@ class RandomPosterize final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomResize TensorTransform. | |||
| /// \note Resize the input image using a randomly selected interpolation mode. | |||
| // the same image aspect ratio. If size has 2 values, it should be (height, width). | |||
| /// \brief Resize the input image using a randomly selected interpolation mode. | |||
| class RandomResize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the resized image. | |||
| /// If size is a single value, the smaller edge of the image will be resized to this value with. | |||
| /// If the size is a single value, the smaller edge of the image will be resized to this value with | |||
| // the same image aspect ratio. If the size has 2 values, it should be (height, width). | |||
| explicit RandomResize(std::vector<int32_t> size); | |||
| /// \brief Destructor. | |||
| ~RandomResize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -537,22 +518,21 @@ class RandomResize final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomResizeWithBBox TensorTransform. | |||
| /// \note Resize the input image using a randomly selected interpolation mode and adjust | |||
| /// \brief Resize the input image using a randomly selected interpolation mode and adjust | |||
| /// bounding boxes accordingly. | |||
| class RandomResizeWithBBox final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the resized image. | |||
| /// If size is a single value, the smaller edge of the image will be resized to this value with | |||
| // the same image aspect ratio. If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, the smaller edge of the image will be resized to this value with | |||
| // the same image aspect ratio. If the size has 2 values, it should be (height, width). | |||
| explicit RandomResizeWithBBox(std::vector<int32_t> size); | |||
| /// \brief Destructor. | |||
| ~RandomResizeWithBBox() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -561,14 +541,13 @@ class RandomResizeWithBBox final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomResizedCrop TensorTransform. | |||
| /// \note Crop the input image to a random size and aspect ratio. | |||
| /// \brief Crop the input image to a random size and aspect ratio. | |||
| class RandomResizedCrop final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the cropped image. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| /// \param[in] scale Range [min, max) of respective size of the original | |||
| /// size to be cropped (default=(0.08, 1.0)). | |||
| /// \param[in] ratio Range [min, max) of aspect ratio to be cropped | |||
| @@ -584,7 +563,7 @@ class RandomResizedCrop final : public TensorTransform { | |||
| ~RandomResizedCrop() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -593,15 +572,14 @@ class RandomResizedCrop final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomResizedCropWithBBox TensorTransform. | |||
| /// \note Crop the input image to a random size and aspect ratio. | |||
| /// \brief Crop the input image to a random size and aspect ratio. | |||
| /// If cropped area is out of bbox, the return bbox will be empty. | |||
| class RandomResizedCropWithBBox final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the cropped image. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| /// \param[in] scale Range [min, max) of respective size of the original | |||
| /// size to be cropped (default=(0.08, 1.0)). | |||
| /// \param[in] ratio Range [min, max) of aspect ratio to be cropped | |||
| @@ -617,7 +595,7 @@ class RandomResizedCropWithBBox final : public TensorTransform { | |||
| ~RandomResizedCropWithBBox() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -626,8 +604,7 @@ class RandomResizedCropWithBBox final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomRotation TensorOp. | |||
| /// \note Rotates the image according to parameters. | |||
| /// \brief Rotate the image according to parameters. | |||
| class RandomRotation final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -635,7 +612,7 @@ class RandomRotation final : public TensorTransform { | |||
| /// \param[in] resample An enum for the mode of interpolation. | |||
| /// \param[in] expand A boolean representing whether the image is expanded after rotation. | |||
| /// \param[in] center A float vector of size 2, representing the x and y center of rotation. | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transform. | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transform | |||
| /// in the output image. If 1 value is provided, it is used for all RGB channels. | |||
| /// If 3 values are provided, it is used to fill R, G, B channels respectively. | |||
| RandomRotation(std::vector<float> degrees, InterpolationMode resample = InterpolationMode::kNearestNeighbour, | |||
| @@ -646,7 +623,7 @@ class RandomRotation final : public TensorTransform { | |||
| ~RandomRotation() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -655,11 +632,10 @@ class RandomRotation final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomSelectSubpolicy TensorTransform. | |||
| /// \note Choose a random sub-policy from a list to be applied on the input image. A sub-policy is a list of tuples | |||
| /// (op, prob), where op is a TensorTransform operation and prob is the probability that this op will be applied. | |||
| /// Once a sub-policy is selected, each op within the sub-policy with be applied in sequence according to its | |||
| /// probability. | |||
| /// \brief Choose a random sub-policy from a list to be applied on the input image. A sub-policy is a list of tuples | |||
| /// (operation, prob), where operation is a TensorTransform operation and prob is the probability that this | |||
| /// operation will be applied. Once a sub-policy is selected, each operation within the sub-policy with be | |||
| /// applied in sequence according to its probability. | |||
| class RandomSelectSubpolicy final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -680,7 +656,7 @@ class RandomSelectSubpolicy final : public TensorTransform { | |||
| ~RandomSelectSubpolicy() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -689,20 +665,20 @@ class RandomSelectSubpolicy final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomSharpness TensorTransform. | |||
| /// \note Tensor operation to perform random sharpness. | |||
| /// \brief Adjust the sharpness of the input image by a fixed or random degree. | |||
| class RandomSharpness final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] degrees A float vector of size 2, representing the starting and ending degree to uniformly. | |||
| /// sample from, to select a degree to adjust sharpness. | |||
| /// \param[in] degrees A float vector of size 2, representing the range of random sharpness | |||
| /// adjustment degrees. It should be in (min, max) format. If min=max, then it is a | |||
| /// single fixed magnitude operation (default = (0.1, 1.9)). | |||
| explicit RandomSharpness(std::vector<float> degrees = {0.1, 1.9}); | |||
| /// \brief Destructor. | |||
| ~RandomSharpness() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -711,20 +687,20 @@ class RandomSharpness final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomSolarize TensorTransform. | |||
| /// \note Invert pixels randomly within specified range. If min=max, it is a single fixed magnitude operation | |||
| /// to inverts all pixel above that threshold. | |||
| /// \brief Invert pixels randomly within a specified range. | |||
| class RandomSolarize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] threshold A vector with two elements specifying the pixel range to invert. | |||
| /// Threshold values should always be in (min, max) format. | |||
| /// If min=max, it will to invert all pixels above min(max). | |||
| explicit RandomSolarize(std::vector<uint8_t> threshold = {0, 255}); | |||
| /// \brief Destructor. | |||
| ~RandomSolarize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -733,8 +709,7 @@ class RandomSolarize final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomVerticalFlip TensorTransform. | |||
| /// \note Tensor operation to perform random vertical flip. | |||
| /// \brief Randomly flip the input image vertically with a given probability. | |||
| class RandomVerticalFlip final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -745,7 +720,7 @@ class RandomVerticalFlip final : public TensorTransform { | |||
| ~RandomVerticalFlip() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -754,8 +729,7 @@ class RandomVerticalFlip final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomVerticalFlipWithBBox TensorTransform. | |||
| /// \note Flip the input image vertically, randomly with a given probability and adjust bounding boxes accordingly. | |||
| /// \brief Randomly flip the input image vertically with a given probability and adjust bounding boxes accordingly. | |||
| class RandomVerticalFlipWithBBox final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -766,7 +740,7 @@ class RandomVerticalFlipWithBBox final : public TensorTransform { | |||
| ~RandomVerticalFlipWithBBox() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -775,8 +749,7 @@ class RandomVerticalFlipWithBBox final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RescaleOperation TensorTransform. | |||
| /// \note Tensor operation to rescale the input image. | |||
| /// \brief Rescale the pixel value of input image. | |||
| class Rescale final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -788,7 +761,7 @@ class Rescale final : public TensorTransform { | |||
| ~Rescale() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -797,14 +770,13 @@ class Rescale final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief ResizeWithBBox TensorTransform. | |||
| /// \note Resize the input image to the given size and adjust bounding boxes accordingly. | |||
| /// \brief Resize the input image to the given size and adjust bounding boxes accordingly. | |||
| class ResizeWithBBox final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size The output size of the resized image. | |||
| /// If size is an integer, smaller edge of the image will be resized to this value with the same image aspect | |||
| /// ratio. If size is a sequence of length 2, it should be (height, width). | |||
| /// If the size is an integer, smaller edge of the image will be resized to this value with the same image aspect | |||
| /// ratio. If the size is a sequence of length 2, it should be (height, width). | |||
| /// \param[in] interpolation An enum for the mode of interpolation (default=InterpolationMode::kLinear). | |||
| explicit ResizeWithBBox(std::vector<int32_t> size, InterpolationMode interpolation = InterpolationMode::kLinear); | |||
| @@ -812,7 +784,7 @@ class ResizeWithBBox final : public TensorTransform { | |||
| ~ResizeWithBBox() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -821,8 +793,7 @@ class ResizeWithBBox final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RgbaToBgr TensorTransform. | |||
| /// \note Changes the input 4 channel RGBA tensor to 3 channel BGR. | |||
| /// \brief Change the format of input tensor from 4-channel RGBA to 3-channel BGR. | |||
| class RGBA2BGR final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -832,13 +803,12 @@ class RGBA2BGR final : public TensorTransform { | |||
| ~RGBA2BGR() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief RgbaToRgb TensorTransform. | |||
| /// \note Changes the input 4 channel RGBA tensor to 3 channel RGB. | |||
| /// \brief Change the input 4 channel RGBA tensor to 3 channel RGB. | |||
| class RGBA2RGB final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -848,23 +818,22 @@ class RGBA2RGB final : public TensorTransform { | |||
| ~RGBA2RGB() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief SoftDvppDecodeRandomCropResizeJpeg TensorTransform. | |||
| /// \note Tensor operation to decode, random crop and resize JPEG image using the simulation algorithm of | |||
| /// Ascend series chip DVPP module. The usage scenario is consistent with SoftDvppDecodeResizeJpeg. | |||
| /// \brief Decode, randomly crop and resize a JPEG image using the simulation algorithm of | |||
| /// Ascend series chip DVPP module. The application scenario is consistent with SoftDvppDecodeResizeJpeg. | |||
| /// The input image size should be in range [32*32, 8192*8192]. | |||
| /// The zoom-out and zoom-in multiples of the image length and width should in the range [1/32, 16]. | |||
| /// The zoom-out and zoom-in multiples of the image length and width should be in the range [1/32, 16]. | |||
| /// Only images with an even resolution can be output. The output of odd resolution is not supported. | |||
| class SoftDvppDecodeRandomCropResizeJpeg final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the resized image. | |||
| /// If size is a single value, smaller edge of the image will be resized to this value with | |||
| /// the same image aspect ratio. If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, smaller edge of the image will be resized to this value with | |||
| /// the same image aspect ratio. If the size has 2 values, it should be (height, width). | |||
| /// \param[in] scale Range [min, max) of respective size of the original | |||
| /// size to be cropped (default=(0.08, 1.0)). | |||
| /// \param[in] ratio Range [min, max) of aspect ratio to be cropped | |||
| @@ -878,7 +847,7 @@ class SoftDvppDecodeRandomCropResizeJpeg final : public TensorTransform { | |||
| ~SoftDvppDecodeRandomCropResizeJpeg() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -887,28 +856,27 @@ class SoftDvppDecodeRandomCropResizeJpeg final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief SoftDvppDecodeResizeJpeg TensorTransform. | |||
| /// \note Tensor operation to decode and resize JPEG image using the simulation algorithm of Ascend series | |||
| /// \brief Decode and resize a JPEG image using the simulation algorithm of Ascend series | |||
| /// chip DVPP module. It is recommended to use this algorithm in the following scenarios: | |||
| /// When training, the DVPP of the Ascend chip is not used, | |||
| /// and the DVPP of the Ascend chip is used during inference, | |||
| /// and the accuracy of inference is lower than the accuracy of training; | |||
| /// and the input image size should be in range [32*32, 8192*8192]. | |||
| /// The zoom-out and zoom-in multiples of the image length and width should in the range [1/32, 16]. | |||
| /// The zoom-out and zoom-in multiples of the image length and width should be in the range [1/32, 16]. | |||
| /// Only images with an even resolution can be output. The output of odd resolution is not supported. | |||
| class SoftDvppDecodeResizeJpeg final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the resized image. | |||
| /// If size is a single value, smaller edge of the image will be resized to this value with | |||
| /// the same image aspect ratio. If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, smaller edge of the image will be resized to this value with | |||
| /// the same image aspect ratio. If the size has 2 values, it should be (height, width). | |||
| explicit SoftDvppDecodeResizeJpeg(std::vector<int32_t> size); | |||
| /// \brief Destructor. | |||
| ~SoftDvppDecodeResizeJpeg() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -917,8 +885,7 @@ class SoftDvppDecodeResizeJpeg final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief SwapRedBlue TensorOp. | |||
| /// \note Swaps the red and blue channels in image. | |||
| /// \brief Swap the red and blue channels of the input image. | |||
| class SwapRedBlue final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -928,35 +895,34 @@ class SwapRedBlue final : public TensorTransform { | |||
| ~SwapRedBlue() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief UniformAugment TensorTransform. | |||
| /// \note Tensor operation to perform randomly selected augmentation. | |||
| /// \brief Randomly perform transformations, as selected from input transform list, on the input tensor. | |||
| class UniformAugment final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] transforms Raw pointer to vector of TensorTransform operations. | |||
| /// \param[in] num_ops An integer representing the number of OPs to be selected and applied. | |||
| /// \param[in] num_ops An integer representing the number of operations to be selected and applied. | |||
| explicit UniformAugment(const std::vector<TensorTransform *> &transforms, int32_t num_ops = 2); | |||
| /// \brief Constructor. | |||
| /// \param[in] transforms Smart pointer to vector of TensorTransform operations. | |||
| /// \param[in] num_ops An integer representing the number of OPs to be selected and applied. | |||
| /// \param[in] num_ops An integer representing the number of operations to be selected and applied. | |||
| explicit UniformAugment(const std::vector<std::shared_ptr<TensorTransform>> &transforms, int32_t num_ops = 2); | |||
| /// \brief Constructor. | |||
| /// \param[in] transforms Object pointer to vector of TensorTransform operations. | |||
| /// \param[in] num_ops An integer representing the number of OPs to be selected and applied. | |||
| /// \param[in] num_ops An integer representing the number of operations to be selected and applied. | |||
| explicit UniformAugment(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, int32_t num_ops = 2); | |||
| /// \brief Destructor. | |||
| ~UniformAugment() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -965,8 +931,7 @@ class UniformAugment final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief VerticalFlip TensorTransform. | |||
| /// \note Flip the input image Vertically. | |||
| /// \brief Flip the input image vertically. | |||
| class VerticalFlip final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -976,7 +941,7 @@ class VerticalFlip final : public TensorTransform { | |||
| ~VerticalFlip() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| @@ -34,17 +34,19 @@ namespace vision { | |||
| /* ##################################### API class ###########################################*/ | |||
| /// \brief Decode and resize JPEG image using the hardware algorithm of | |||
| /// Ascend series chip DVPP module. | |||
| class DvppDecodeResizeJpeg final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] resize A vector of int value for each dimension, w.r.t H,W order. | |||
| /// \param[in] resize A vector of int value for each dimension, with respect to H,W order. | |||
| explicit DvppDecodeResizeJpeg(std::vector<uint32_t> resize); | |||
| /// \brief Destructor. | |||
| ~DvppDecodeResizeJpeg() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -55,18 +57,20 @@ class DvppDecodeResizeJpeg final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Decode, resize and crop JPEG image using the hardware algorithm of | |||
| /// Ascend series chip DVPP module. | |||
| class DvppDecodeResizeCropJpeg final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] crop A vector of int value for each dimension after final crop, w.r.t H,W order. | |||
| /// \param[in] resize A vector of int value for each dimension after resize, w.r.t H,W order. | |||
| /// \param[in] crop A vector of int value for each dimension after final cropping, with respect to H,W order. | |||
| /// \param[in] resize A vector of int value for each dimension after resizing, with respect to H,W order. | |||
| explicit DvppDecodeResizeCropJpeg(std::vector<uint32_t> crop, std::vector<uint32_t> resize); | |||
| /// \brief Destructor. | |||
| ~DvppDecodeResizeCropJpeg() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -77,6 +81,8 @@ class DvppDecodeResizeCropJpeg final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Decode PNG image using the hardware algorithm of | |||
| /// Ascend series chip DVPP module. | |||
| class DvppDecodePng final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -86,7 +92,7 @@ class DvppDecodePng final : public TensorTransform { | |||
| ~DvppDecodePng() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -35,18 +35,17 @@ namespace vision { | |||
| // Forward Declarations | |||
| class RotateOperation; | |||
| /// \brief Affine TensorTransform. | |||
| /// \note Apply affine transform on input image. | |||
| /// \brief Apply affine transform on the input image. | |||
| class Affine final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] degrees The degrees to rotate the image by. | |||
| /// \param[in] translation The value representing vertical and horizontal translation (default = {0.0, 0.0}). | |||
| /// The first value represent the x axis translation while the second represents y axis translation. | |||
| /// \param[in] degrees The degrees to rotate the image. | |||
| /// \param[in] translation The values representing vertical and horizontal translation (default = {0.0, 0.0}). | |||
| /// The first value represents the x axis translation while the second represents the y axis translation. | |||
| /// \param[in] scale The scaling factor for the image (default = 0.0). | |||
| /// \param[in] shear A float vector of size 2, representing the shear degrees (default = {0.0, 0.0}). | |||
| /// \param[in] interpolation An enum for the mode of interpolation. | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transform | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transformation | |||
| /// in the output image. If 1 value is provided, it is used for all RGB channels. | |||
| /// If 3 values are provided, it is used to fill R, G, B channels respectively. | |||
| explicit Affine(float_t degrees, const std::vector<float> &translation = {0.0, 0.0}, float scale = 0.0, | |||
| @@ -57,7 +56,7 @@ class Affine final : public TensorTransform { | |||
| /// \brief Destructor. | |||
| ~Affine() = default; | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -66,21 +65,20 @@ class Affine final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief CenterCrop TensorTransform. | |||
| /// \note Crops the input image at the center to the given size. | |||
| /// \brief Crop the input image at the center to the given size. | |||
| class CenterCrop final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the cropped image. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| explicit CenterCrop(std::vector<int32_t> size); | |||
| /// \brief Destructor. | |||
| ~CenterCrop() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -92,7 +90,7 @@ class CenterCrop final : public TensorTransform { | |||
| }; | |||
| /// \brief RGB2BGR TensorTransform. | |||
| /// \notes Convert RGB image to BGR image | |||
| /// \notes Convert the format of input image from RGB to BGR. | |||
| class RGB2BGR final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -102,13 +100,14 @@ class RGB2BGR final : public TensorTransform { | |||
| ~RGB2BGR() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief RGB2GRAY TensorTransform. | |||
| /// \note Convert RGB image or color image to grayscale image. | |||
| /// \brief Convert a RGB image or color image to a grayscale one. | |||
| class RGB2GRAY final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -118,27 +117,26 @@ class RGB2GRAY final : public TensorTransform { | |||
| ~RGB2GRAY() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| }; | |||
| /// \brief Crop TensorTransform. | |||
| /// \note Crop an image based on location and crop size. | |||
| /// \brief Crop an image based on location and crop size. | |||
| class Crop final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] coordinates Starting location of crop. Must be a vector of two values, in the form of {x_coor, y_coor}. | |||
| /// \param[in] size Size of the cropped area. | |||
| /// If size is a single value, a square crop of size (size, size) is returned. | |||
| /// If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, a squared crop of size (size, size) is returned. | |||
| /// If the size has 2 values, it should be (height, width). | |||
| Crop(std::vector<int32_t> coordinates, std::vector<int32_t> size); | |||
| /// \brief Destructor. | |||
| ~Crop() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -147,19 +145,18 @@ class Crop final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Decode TensorTransform. | |||
| /// \note Decode the input image in RGB mode. | |||
| /// \brief Decode the input image in RGB mode. | |||
| class Decode final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] rgb A boolean of whether to decode in RGB mode or not. | |||
| /// \param[in] rgb A boolean indicating whether to decode the image in RGB mode or not. | |||
| explicit Decode(bool rgb = true); | |||
| /// \brief Destructor. | |||
| ~Decode() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -170,12 +167,11 @@ class Decode final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief GaussianBlur TensorTransform. | |||
| /// \notes Blur the input image with specified Gaussian kernel. | |||
| /// \brief Blur the input image with the specified Gaussian kernel. | |||
| class GaussianBlur final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] kernel_size A vector of Gaussian kernel size for width and height. The values must be positive and odd. | |||
| /// \param[in] kernel_size A vector of Gaussian kernel size for width and height. The value must be positive and odd. | |||
| /// \param[in] sigma A vector of Gaussian kernel standard deviation sigma for width and height. The values must be | |||
| /// positive. Using default value 0 means to calculate the sigma according to the kernel size. | |||
| GaussianBlur(const std::vector<int32_t> &kernel_size, const std::vector<float> &sigma = {0., 0.}); | |||
| @@ -184,7 +180,7 @@ class GaussianBlur final : public TensorTransform { | |||
| ~GaussianBlur() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -193,14 +189,13 @@ class GaussianBlur final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Normalize TensorTransform. | |||
| /// \note Normalize the input image with respect to mean and standard deviation. | |||
| /// \brief Normalize the input image with respect to mean and standard deviation. | |||
| class Normalize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] mean A vector of mean values for each channel, w.r.t channel order. | |||
| /// \param[in] mean A vector of mean values for each channel, with respect to channel order. | |||
| /// The mean values must be in range [0.0, 255.0]. | |||
| /// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order. | |||
| /// \param[in] std A vector of standard deviations for each channel, with respect to channel order. | |||
| /// The standard deviation values must be in range (0.0, 255.0]. | |||
| Normalize(std::vector<float> mean, std::vector<float> std); | |||
| @@ -208,7 +203,7 @@ class Normalize final : public TensorTransform { | |||
| ~Normalize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -219,25 +214,24 @@ class Normalize final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief RandomAffine TensorTransform. | |||
| /// \note Applies a Random Affine transformation on input image in RGB or Greyscale mode. | |||
| /// \brief Apply a Random Affine transformation on the input image in RGB or Greyscale mode. | |||
| class RandomAffine final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] degrees A float vector of size 2, representing the starting and ending degree. | |||
| /// \param[in] translate_range A float vector of size 2 or 4, representing percentages of translation on x and y axes. | |||
| /// If size is 2, (min_dx, max_dx, 0, 0). | |||
| /// if size is 4, (min_dx, max_dx, min_dy, max_dy), | |||
| /// If the size is 2, (min_dx, max_dx, 0, 0). | |||
| /// If the size is 4, (min_dx, max_dx, min_dy, max_dy), | |||
| /// all values are in range [-1, 1]. | |||
| /// \param[in] scale_range A float vector of size 2, representing the starting and ending scales in the range. | |||
| /// \param[in] shear_ranges A float vector of size 2 or 4, representing the starting and ending shear degrees | |||
| /// vertically and horizontally. | |||
| /// If size is 2, (min_shear_x, max_shear_x, 0, 0), | |||
| /// if size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y). | |||
| /// If the size is 2, (min_shear_x, max_shear_x, 0, 0), | |||
| /// if the size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y). | |||
| /// \param[in] interpolation An enum for the mode of interpolation. | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transform | |||
| /// in the output image. If 1 value is provided, it is used for all RGB channels. | |||
| /// If 3 values are provided, it is used to fill R, G, B channels respectively. | |||
| /// If 3 values are provided, it is used to fill R, G and B channels respectively. | |||
| explicit RandomAffine(const std::vector<float_t> °rees, | |||
| const std::vector<float_t> &translate_range = {0.0, 0.0, 0.0, 0.0}, | |||
| const std::vector<float_t> &scale_range = {1.0, 1.0}, | |||
| @@ -248,7 +242,7 @@ class RandomAffine final : public TensorTransform { | |||
| /// \brief Destructor. | |||
| ~RandomAffine() = default; | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -257,14 +251,13 @@ class RandomAffine final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Resize TensorTransform. | |||
| /// \note Resize the input image to the given size. | |||
| /// \brief Resize the input image to the given size. | |||
| class Resize final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| /// \param[in] size A vector representing the output size of the resized image. | |||
| /// If size is a single value, the image will be resized to this value with | |||
| /// the same image aspect ratio. If size has 2 values, it should be (height, width). | |||
| /// If the size is a single value, the image will be resized to this value with | |||
| /// the same image aspect ratio. If the size has 2 values, it should be (height, width). | |||
| /// \param[in] interpolation An enum for the mode of interpolation. | |||
| explicit Resize(std::vector<int32_t> size, InterpolationMode interpolation = InterpolationMode::kLinear); | |||
| @@ -272,7 +265,7 @@ class Resize final : public TensorTransform { | |||
| ~Resize() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -283,8 +276,7 @@ class Resize final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief ResizePreserveAR TensorTransform. | |||
| /// \note Keep the original picture ratio and fill the rest. | |||
| /// \brief Keep the original picture ratio and fills the rest. | |||
| class ResizePreserveAR final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -297,7 +289,7 @@ class ResizePreserveAR final : public TensorTransform { | |||
| ~ResizePreserveAR() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -306,8 +298,7 @@ class ResizePreserveAR final : public TensorTransform { | |||
| std::shared_ptr<Data> data_; | |||
| }; | |||
| /// \brief Rotate TensorTransform. | |||
| /// \note Rotate the input image according to parameters. | |||
| /// \brief Rotate the input image according to parameters. | |||
| class Rotate final : public TensorTransform { | |||
| public: | |||
| /// \brief Constructor. | |||
| @@ -318,7 +309,7 @@ class Rotate final : public TensorTransform { | |||
| /// \param[in] resample An enum for the mode of interpolation. | |||
| /// \param[in] expand A boolean representing whether the image is expanded after rotation. | |||
| /// \param[in] center A float vector of size 2, representing the x and y center of rotation. | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transform. | |||
| /// \param[in] fill_value A vector representing the value to fill the area outside the transform | |||
| /// in the output image. If 1 value is provided, it is used for all RGB channels. | |||
| /// If 3 values are provided, it is used to fill R, G, B channels respectively. | |||
| Rotate(float degrees, InterpolationMode resample = InterpolationMode::kNearestNeighbour, bool expand = false, | |||
| @@ -328,7 +319,7 @@ class Rotate final : public TensorTransform { | |||
| ~Rotate() = default; | |||
| protected: | |||
| /// \brief Function to convert TensorTransform object into a TensorOperation object. | |||
| /// \brief The function to convert a TensorTransform object into a TensorOperation object. | |||
| /// \return Shared pointer to TensorOperation object. | |||
| std::shared_ptr<TensorOperation> Parse() override; | |||
| @@ -17,6 +17,7 @@ | |||
| #define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H | |||
| #include <string> | |||
| #include <memory> | |||
| #include "minddata/dataset/core/tensor.h" | |||
| #include "minddata/dataset/kernels/image/image_utils.h" | |||
| #include "minddata/dataset/kernels/tensor_op.h" | |||
| @@ -36,9 +37,21 @@ class ResizeWithBBoxOp : public ResizeOp { | |||
| void Print(std::ostream &out) const override { out << Name() << ": " << size1_ << " " << size2_; } | |||
| // Use in pipeline mode | |||
| Status Compute(const TensorRow &input, TensorRow *output) override; | |||
| // Use in execute mode | |||
| // ResizeWithBBoxOp is inherited from ResizeOp and this function has been overridden by ResizeOp, | |||
| // thus we need to change the behaviour back to basic class (TensorOp). | |||
| Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override { | |||
| return TensorOp::Compute(input, output); | |||
| } | |||
| std::string Name() const override { return kResizeWithBBoxOp; } | |||
| uint32_t NumInput() override { return 2; } | |||
| uint32_t NumOutput() override { return 2; } | |||
| }; | |||
| } // namespace dataset | |||
| } // namespace mindspore | |||
| @@ -437,15 +437,16 @@ class ShuffleDataset : public Dataset { | |||
| ~ShuffleDataset() = default; | |||
| }; | |||
| /// \brief Function to create a SchemaObj | |||
| /// \param[in] schema_file Path of schema file | |||
| /// \note This api exists because std::string will constrained by ABI compile macro but char don't. | |||
| /// \return Shared pointer to the current schema | |||
| /// \brief Function to create a SchemaObj. | |||
| /// \param[in] schema_file Path of schema file. | |||
| /// \note The reason for using this API is that std::string will be constrained by the | |||
| /// compiler option '_GLIBCXX_USE_CXX11_ABI' while char is free of this restriction. | |||
| /// \return Shared pointer to the current schema. | |||
| std::shared_ptr<SchemaObj> SchemaCharIF(const std::vector<char> &schema_file); | |||
| /// \brief Function to create a SchemaObj | |||
| /// \param[in] schema_file Path of schema file | |||
| /// \return Shared pointer to the current schema | |||
| /// \brief Function to create a SchemaObj. | |||
| /// \param[in] schema_file Path of schema file. | |||
| /// \return Shared pointer to the current schema. | |||
| inline std::shared_ptr<SchemaObj> Schema(const std::string &schema_file = "") { | |||
| return SchemaCharIF(StringToChar(schema_file)); | |||
| } | |||
| @@ -38,6 +38,12 @@ class TruncateSequencePairOp : public TensorOp { | |||
| std::string Name() const override { return kTruncateSequencePairOp; } | |||
| // Unknown input size | |||
| uint32_t NumInput() override { return -1; } | |||
| // Unknown output size | |||
| uint32_t NumOutput() override { return -1; } | |||
| private: | |||
| dsize_t max_length_; | |||
| }; | |||
| @@ -1573,12 +1573,25 @@ class Dataset: | |||
| return self.dataset_size | |||
| def set_dynamic_columns(self, columns=None): | |||
| """ | |||
| Set dynamic shape information of source data, it should be set after the pipeline is defined. | |||
| Args: | |||
| columns (dict): A dict contains shape information of each column in dataset. | |||
| The value of shape[i] is :py:obj:`None` indicates that the data length of shape[i] is dynamic. | |||
| """ | |||
| if not isinstance(columns, dict): | |||
| raise TypeError("Pass a dict to set dynamic shape, example: {\"data1\": [16, None, 256]}") | |||
| self.dynamic_setting[0] = True | |||
| self.dynamic_setting[1] = columns | |||
| def dynamic_min_max_shapes(self): | |||
| """ | |||
| Get minimum and maximum data length of dynamic source data, for graph compilation of ME. | |||
| Returns: | |||
| lists, min_shapes, max_shapes of source data. | |||
| """ | |||
| if self.saved_min_shapes is None or self.saved_max_shapes is None: | |||
| self.saved_output_shapes, self.saved_min_shapes, self.saved_max_shapes = self._dynamic_output_shapes() | |||
| return self.saved_min_shapes, self.saved_max_shapes | |||
| @@ -1591,7 +1604,7 @@ class Dataset: | |||
| lists, dynamic_shapes, min_shapes, max_shapes of source data. | |||
| """ | |||
| if not self.dynamic_setting[1]: | |||
| raise RuntimeError("dynamic_columns is not set, call set_dynamic_columns() first.") | |||
| raise RuntimeError("dynamic_columns is not set, call set_dynamic_columns() by final Dataset Op.") | |||
| if self.saved_output_shapes is not None and self.saved_min_shapes is not None and \ | |||
| self.saved_max_shapes is not None: | |||
| @@ -1648,12 +1661,16 @@ class Dataset: | |||
| dynamic_shapes.append(dynamic_shape.tolist()) | |||
| else: | |||
| # Also append fix shape to keep order of column shape | |||
| if col in dynamic_columns: | |||
| logger.warning("column [" + col + "] has no dynamic shape but set by set_dynamic_columns()") | |||
| fix_shape = list(list(shape_set)[0]) | |||
| max_shapes.append(fix_shape) | |||
| min_shapes.append(fix_shape) | |||
| dynamic_shapes.append(fix_shape) | |||
| if col in dynamic_columns: | |||
| logger.warning("column [" + col + "] has no dynamic shape but set by set_dynamic_columns()") | |||
| # Set min shape to 1 due to unknown shuffle | |||
| min_shapes[-1] = np.where(np.equal(dynamic_columns[col], None), 1, fix_shape).tolist() | |||
| # Set dynamic dim to -1 for ME | |||
| dynamic_shapes[-1] = np.where(np.equal(dynamic_columns[col], None), -1, fix_shape).tolist() | |||
| return dynamic_shapes, min_shapes, max_shapes | |||
| def num_classes(self): | |||
| @@ -3109,7 +3126,7 @@ class ImageFolderDataset(MappableDataset): | |||
| decode (bool, optional): Decode the images after reading (default=False). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, `num_samples` reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -3229,7 +3246,7 @@ class MnistDataset(MappableDataset): | |||
| sampler (Sampler, optional): Object used to choose samples from the | |||
| dataset (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). | |||
| When this argument is specified, `num_samples` reflects the max sample number of per shard. | |||
| When this argument is specified, `num_samples` reflects the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within `num_shards` (default=None). This | |||
| argument can only be specified when `num_shards` is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -3347,7 +3364,7 @@ class MindDataset(MappableDataset): | |||
| - Shuffle.INFILE: Keep the file sequence the same but shuffle the data within each file. | |||
| num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). | |||
| When this argument is specified, 'num_samples' reflects the max sample number of per shard. | |||
| When this argument is specified, 'num_samples' reflects the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| sampler (Sampler, optional): Object used to choose samples from the | |||
| @@ -3786,8 +3803,8 @@ class GeneratorDataset(MappableDataset): | |||
| sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible | |||
| input is required (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). | |||
| Random accessible input is required. When this argument is specified, `num_samples`s reflects the max sample | |||
| number of per shard. | |||
| Random accessible input is required. When this argument is specified, `num_samples` reflects the maximum | |||
| sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This argument must be specified only | |||
| when num_shards is also specified. Random accessible input is required. | |||
| python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This | |||
| @@ -4007,12 +4024,13 @@ class TFRecordDataset(SourceDataset): | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, `num_samples` reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| shard_equal_rows (bool, optional): Get equal rows for all shards(default=False). If shard_equal_rows | |||
| is false, number of rows of each shard may be not equal. This | |||
| argument should only be specified when num_shards is also specified. | |||
| is false, number of rows of each shard may be not equal, and may lead to a failure in distributed training. | |||
| When the number of samples of per TFRecord file are not equal, it is suggested to set to true. | |||
| This argument should only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| (default=None, which means no cache is used). | |||
| @@ -4200,7 +4218,7 @@ class Cifar10Dataset(MappableDataset): | |||
| dataset (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, `num_samples` reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -4324,7 +4342,7 @@ class Cifar100Dataset(MappableDataset): | |||
| dataset (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, 'num_samples' reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -4437,7 +4455,7 @@ class RandomDataset(SourceDataset): | |||
| (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, 'num_samples' reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| """ | |||
| @@ -4600,7 +4618,7 @@ class VOCDataset(MappableDataset): | |||
| (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, `num_samples` reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -4771,7 +4789,7 @@ class CocoDataset(MappableDataset): | |||
| :py:obj:`[category_id, dtype=uint32]`, :py:obj:`[iscrowd, dtype=uint32]`. | |||
| - task = :py:obj:`Stuff`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[segmentation,dtype=float32]`, \ | |||
| :py:obj:`[iscrowd,dtype=uint32]`. | |||
| - task = :py:obj:`Keypoint, output columns: :py:obj:`[image, dtype=uint8]`, \ | |||
| - task = :py:obj:`Keypoint`, output columns: :py:obj:`[image, dtype=uint8]`, \ | |||
| :py:obj:`[keypoints, dtype=float32]`, :py:obj:`[num_keypoints, dtype=uint32]`. | |||
| - task = :py:obj:`Panoptic`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[bbox, dtype=float32]`, \ | |||
| :py:obj:`[category_id, dtype=uint32]`, :py:obj:`[iscrowd, dtype=uint32]`, :py:obj:`[area, dtype=uint32]`. | |||
| @@ -4792,7 +4810,7 @@ class CocoDataset(MappableDataset): | |||
| (default=None, expected order behavior shown in the table). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, `num_samples` reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -4973,7 +4991,7 @@ class CelebADataset(MappableDataset): | |||
| (default=None, will include all images). | |||
| num_shards (int, optional): Number of shards that the dataset will be divided | |||
| into (default=None). When this argument is specified, `num_samples` reflects | |||
| the max sample number of per shard. | |||
| the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within `num_shards` (default=None). This | |||
| argument can only be specified when `num_shards` is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -5183,7 +5201,7 @@ class CLUEDataset(SourceDataset): | |||
| - Shuffle.FILES: Shuffle files only. | |||
| num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). | |||
| When this argument is specified, `num_samples` reflects the max sample number of per shard. | |||
| When this argument is specified, `num_samples` reflects the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -5268,7 +5286,7 @@ class CSVDataset(SourceDataset): | |||
| - Shuffle.FILES: Shuffle files only. | |||
| num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). | |||
| When this argument is specified, `num_samples` reflects the max sample number of per shard. | |||
| When this argument is specified, `num_samples` reflects the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -5323,7 +5341,7 @@ class TextFileDataset(SourceDataset): | |||
| - Shuffle.FILES: Shuffle files only. | |||
| num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). | |||
| When this argument is specified, `num_samples` reflects the max sample number of per shard. | |||
| When this argument is specified, `num_samples` reflects the maximum sample number of per shard. | |||
| shard_id (int, optional): The shard ID within num_shards (default=None). This | |||
| argument can only be specified when num_shards is also specified. | |||
| cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. | |||
| @@ -279,4 +279,19 @@ TEST_F(MindDataTestExecute, TestRotate) { | |||
| Status rc = transform(image, &image); | |||
| EXPECT_EQ(rc, Status::OK()); | |||
| } | |||
| } | |||
| TEST_F(MindDataTestExecute, TestResizeWithBBox) { | |||
| auto image = ReadFileToTensor("data/dataset/apple.jpg"); | |||
| std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>(); | |||
| std::shared_ptr<TensorTransform> resizewithbbox_op = | |||
| std::make_shared<vision::ResizeWithBBox>(std::vector<int32_t>{250, 500}); | |||
| // Test Compute(Tensor, Tensor) method of ResizeWithBBox | |||
| auto transform = Execute({decode_op, resizewithbbox_op}); | |||
| // Expect fail since Compute(Tensor, Tensor) is not a valid behaviour for this Op, | |||
| // while Compute(TensorRow, TensorRow) is the correct one. | |||
| Status rc = transform(image, &image); | |||
| EXPECT_FALSE(rc.IsOk()); | |||
| } | |||
| @@ -24,7 +24,7 @@ def generator0(): | |||
| def test_get_dynamic_min_max_shapes_0(): | |||
| logger.info("Test dynamic_min_max_shapes with dynamic shape columns") | |||
| logger.info("Test dynamic_min_max_shapes with dynamic shape columns.") | |||
| dataset = ds.GeneratorDataset(generator0, ["data1", "data2", "data3"]) | |||
| @@ -47,7 +47,7 @@ def generator1(): | |||
| def test_get_dynamic_min_max_shapes_1(): | |||
| logger.info("Test dynamic_min_max_shapes with dynamic shape column and fix shape column") | |||
| logger.info("Test dynamic_min_max_shapes with dynamic shape column and fix shape column.") | |||
| dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) | |||
| @@ -66,7 +66,7 @@ def test_get_dynamic_min_max_shapes_1(): | |||
| def test_get_dynamic_min_max_shapes_2(): | |||
| logger.info("Test dynamic_min_max_shapes with all dynamic config") | |||
| logger.info("Test dynamic_min_max_shapes with setting all columns to dynamic.") | |||
| dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) | |||
| @@ -88,7 +88,7 @@ def generator2(): | |||
| def test_get_dynamic_min_max_shapes_3(): | |||
| logger.info("Test dynamic_min_max_shapes with only config dynamic column") | |||
| logger.info("Test dynamic_min_max_shapes only config dynamic column.") | |||
| dataset = ds.GeneratorDataset(generator2, ["data1", "data2"]) | |||
| @@ -100,14 +100,57 @@ def test_get_dynamic_min_max_shapes_3(): | |||
| min_shapes, max_shapes = dataset.dynamic_min_max_shapes() | |||
| # check result | |||
| # column with fix shape will be also appended to shapes list | |||
| # column with fixed shape will also be appended to shapes list | |||
| np.testing.assert_array_equal(min_shapes, [[16, 1, 83], [5, 5]]) | |||
| np.testing.assert_array_equal(max_shapes, [[16, 99, 83], [5, 5]]) | |||
| np.testing.assert_array_equal(dynamic_shapes, [[16, -1, 83], [5, 5]]) | |||
| def test_get_dynamic_min_max_shapes_4(): | |||
| logger.info("Test dynamic_min_max_shapes with unexpected column setting") | |||
| logger.info("Test dynamic_min_max_shapes with dynamic setting for column with fixed shape.") | |||
| dataset = ds.GeneratorDataset(generator2, ["data1", "data2"]) | |||
| # only dynamic shape is required to config | |||
| dataset.set_dynamic_columns(columns={"data1": [16, None, 83], "data2": [None, 5]}) | |||
| # get dynamic information | |||
| dynamic_shapes = dataset.output_shapes() | |||
| min_shapes, max_shapes = dataset.dynamic_min_max_shapes() | |||
| # check result | |||
| # column with fixed shape will also be appended to shapes list | |||
| np.testing.assert_array_equal(min_shapes, [[16, 1, 83], [1, 5]]) | |||
| np.testing.assert_array_equal(max_shapes, [[16, 99, 83], [5, 5]]) | |||
| np.testing.assert_array_equal(dynamic_shapes, [[16, -1, 83], [-1, 5]]) | |||
| def test_get_dynamic_min_max_shapes_5(): | |||
| logger.info("Test dynamic_min_max_shapes with NumpySlicesDataset.") | |||
| np_data = [ | |||
| [[1, 2], [3, 4]], | |||
| [[5, 6], [7, 8]], | |||
| [[9, 10], [11, 12]], | |||
| [[13, 14], [15, 16]] | |||
| ] | |||
| dataset = ds.NumpySlicesDataset(np_data, column_names=["col1"]) | |||
| dataset.set_dynamic_columns(columns={"col1": [2, None]}) | |||
| # get dynamic information | |||
| dynamic_shapes = dataset.output_shapes() | |||
| min_shapes, max_shapes = dataset.dynamic_min_max_shapes() | |||
| # check result | |||
| # column with fixed shape will also be appended to shapes list | |||
| np.testing.assert_array_equal(min_shapes, [[2, 1]]) | |||
| np.testing.assert_array_equal(max_shapes, [[2, 2]]) | |||
| np.testing.assert_array_equal(dynamic_shapes, [[2, -1]]) | |||
| def test_get_dynamic_min_max_shapes_6(): | |||
| logger.info("Test dynamic_min_max_shapes with unexpected column setting.") | |||
| dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) | |||
| @@ -120,7 +163,7 @@ def test_get_dynamic_min_max_shapes_4(): | |||
| # dynamic column is not set | |||
| dataset.set_dynamic_columns(columns=dict()) | |||
| dataset.dynamic_min_max_shapes() | |||
| assert "dynamic_columns is not set, call set_dynamic_columns() first" in str(info.value) | |||
| assert "dynamic_columns is not set, call set_dynamic_columns()" in str(info.value) | |||
| with pytest.raises(RuntimeError) as info: | |||
| # dynamic column is not set | |||
| @@ -153,4 +196,6 @@ if __name__ == "__main__": | |||
| test_get_dynamic_min_max_shapes_2() | |||
| test_get_dynamic_min_max_shapes_3() | |||
| test_get_dynamic_min_max_shapes_4() | |||
| test_get_dynamic_min_max_shapes_5() | |||
| test_get_dynamic_min_max_shapes_6() | |||