From: @tiancixiao Reviewed-by: @heleiwang,@liucunwei Signed-off-by: @liucunweitags/v1.2.0-rc1
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -52,13 +52,14 @@ Status SoftDvppDecodeRandomCropResizeJpegOp::Compute(const std::shared_ptr<Tenso | |||||
| std::shared_ptr<Tensor> *output) { | std::shared_ptr<Tensor> *output) { | ||||
| IO_CHECK(input, output); | IO_CHECK(input, output); | ||||
| if (!IsNonEmptyJPEG(input)) { | if (!IsNonEmptyJPEG(input)) { | ||||
| RETURN_STATUS_UNEXPECTED("SoftDvppDecodeRandomCropResizeJpeg only support process jpeg image."); | |||||
| RETURN_STATUS_UNEXPECTED("SoftDvppDecodeRandomCropResizeJpeg: only support processing raw jpeg image."); | |||||
| } | } | ||||
| SoftDpCropInfo crop_info; | SoftDpCropInfo crop_info; | ||||
| RETURN_IF_NOT_OK(GetCropInfo(input, &crop_info)); | RETURN_IF_NOT_OK(GetCropInfo(input, &crop_info)); | ||||
| try { | try { | ||||
| unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); | unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); | ||||
| CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, "The input image buffer is empty."); | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, | |||||
| "SoftDvppDecodeRandomCropResizeJpeg: the input image buffer is empty."); | |||||
| SoftDpProcsessInfo info; | SoftDpProcsessInfo info; | ||||
| info.input_buffer = static_cast<uint8_t *>(buffer); | info.input_buffer = static_cast<uint8_t *>(buffer); | ||||
| info.input_buffer_size = input->SizeInBytes(); | info.input_buffer_size = input->SizeInBytes(); | ||||
| @@ -69,14 +70,14 @@ Status SoftDvppDecodeRandomCropResizeJpegOp::Compute(const std::shared_ptr<Tenso | |||||
| info.output_buffer_size = target_width_ * target_height_ * 3; | info.output_buffer_size = target_width_ * target_height_ * 3; | ||||
| info.is_v_before_u = true; | info.is_v_before_u = true; | ||||
| int ret = DecodeAndCropAndResizeJpeg(&info, crop_info); | int ret = DecodeAndCropAndResizeJpeg(&info, crop_info); | ||||
| std::string error_info("Soft dvpp DecodeAndResizeJpeg failed with return code: "); | |||||
| error_info += std::to_string(ret); | |||||
| std::string error_info("SoftDvppDecodeRandomCropResizeJpeg: failed with return code: "); | |||||
| error_info += std::to_string(ret) + ", please check the log information for more details."; | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info); | CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info); | ||||
| std::shared_ptr<CVTensor> cv_tensor = nullptr; | std::shared_ptr<CVTensor> cv_tensor = nullptr; | ||||
| RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor)); | RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor)); | ||||
| *output = std::static_pointer_cast<Tensor>(cv_tensor); | *output = std::static_pointer_cast<Tensor>(cv_tensor); | ||||
| } catch (const cv::Exception &e) { | } catch (const cv::Exception &e) { | ||||
| std::string error = "Error in SoftDvppDecodeRandomCropResizeJpegOp:" + std::string(e.what()); | |||||
| std::string error = "SoftDvppDecodeRandomCropResizeJpeg:" + std::string(e.what()); | |||||
| RETURN_STATUS_UNEXPECTED(error); | RETURN_STATUS_UNEXPECTED(error); | ||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -28,11 +28,11 @@ namespace dataset { | |||||
| Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { | Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { | ||||
| IO_CHECK(input, output); | IO_CHECK(input, output); | ||||
| if (!IsNonEmptyJPEG(input)) { | if (!IsNonEmptyJPEG(input)) { | ||||
| RETURN_STATUS_UNEXPECTED("SoftDvppDecodeReiszeJpegOp only support process jpeg image."); | |||||
| RETURN_STATUS_UNEXPECTED("SoftDvppDecodeReiszeJpeg: only support processing raw jpeg image."); | |||||
| } | } | ||||
| try { | try { | ||||
| unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); | unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); | ||||
| CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, "The input image buffer is empty."); | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(buffer != nullptr, "SoftDvppDecodeReiszeJpeg: the input image buffer is empty."); | |||||
| SoftDpProcsessInfo info; | SoftDpProcsessInfo info; | ||||
| info.input_buffer = static_cast<uint8_t *>(buffer); | info.input_buffer = static_cast<uint8_t *>(buffer); | ||||
| info.input_buffer_size = input->SizeInBytes(); | info.input_buffer_size = input->SizeInBytes(); | ||||
| @@ -43,11 +43,11 @@ Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, | |||||
| if (target_width_ == 0) { | if (target_width_ == 0) { | ||||
| if (input_h < input_w) { | if (input_h < input_w) { | ||||
| CHECK_FAIL_RETURN_UNEXPECTED(input_h != 0, "The input height is 0"); | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(input_h != 0, "SoftDvppDecodeReiszeJpeg: the input height is 0."); | |||||
| info.output_height = target_height_; | info.output_height = target_height_; | ||||
| info.output_width = static_cast<int>(std::lround(static_cast<float>(input_w) / input_h * info.output_height)); | info.output_width = static_cast<int>(std::lround(static_cast<float>(input_w) / input_h * info.output_height)); | ||||
| } else { | } else { | ||||
| CHECK_FAIL_RETURN_UNEXPECTED(input_w != 0, "The input width is 0"); | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(input_w != 0, "SoftDvppDecodeReiszeJpeg: the input width is 0."); | |||||
| info.output_width = target_height_; | info.output_width = target_height_; | ||||
| info.output_height = static_cast<int>(std::lround(static_cast<float>(input_h) / input_w * info.output_width)); | info.output_height = static_cast<int>(std::lround(static_cast<float>(input_h) / input_w * info.output_width)); | ||||
| } | } | ||||
| @@ -62,14 +62,14 @@ Status SoftDvppDecodeResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, | |||||
| info.is_v_before_u = true; | info.is_v_before_u = true; | ||||
| int ret = DecodeAndResizeJpeg(&info); | int ret = DecodeAndResizeJpeg(&info); | ||||
| std::string error_info("Soft dvpp DecodeAndResizeJpeg failed with return code: "); | |||||
| error_info += std::to_string(ret); | |||||
| std::string error_info("SoftDvppDecodeReiszeJpeg: failed with return code: "); | |||||
| error_info += std::to_string(ret) + ", please check the log information for more details."; | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info); | CHECK_FAIL_RETURN_UNEXPECTED(ret == 0, error_info); | ||||
| std::shared_ptr<CVTensor> cv_tensor = nullptr; | std::shared_ptr<CVTensor> cv_tensor = nullptr; | ||||
| RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor)); | RETURN_IF_NOT_OK(CVTensor::CreateFromMat(out_rgb_img, &cv_tensor)); | ||||
| *output = std::static_pointer_cast<Tensor>(cv_tensor); | *output = std::static_pointer_cast<Tensor>(cv_tensor); | ||||
| } catch (const cv::Exception &e) { | } catch (const cv::Exception &e) { | ||||
| std::string error = "Error in SoftDvppDecodeResizeJpegOp:" + std::string(e.what()); | |||||
| std::string error = "SoftDvppDecodeResizeJpeg:" + std::string(e.what()); | |||||
| RETURN_STATUS_UNEXPECTED(error); | RETURN_STATUS_UNEXPECTED(error); | ||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -82,7 +82,7 @@ Status SoftDvppDecodeResizeJpegOp::OutputShape(const std::vector<TensorShape> &i | |||||
| TensorShape out({-1, -1, 3}); // we don't know what is output image size, but we know it should be 3 channels | TensorShape out({-1, -1, 3}); // we don't know what is output image size, but we know it should be 3 channels | ||||
| if (inputs[0].Rank() == 1) outputs.emplace_back(out); | if (inputs[0].Rank() == 1) outputs.emplace_back(out); | ||||
| if (!outputs.empty()) return Status::OK(); | if (!outputs.empty()) return Status::OK(); | ||||
| return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape"); | |||||
| return Status(StatusCode::kMDUnexpectedError, "SoftDvppDecodeReiszeJpeg: input has a wrong shape."); | |||||
| } | } | ||||
| } // namespace dataset | } // namespace dataset | ||||
| @@ -1512,6 +1512,14 @@ SoftDvppDecodeRandomCropResizeJpegOperation::SoftDvppDecodeRandomCropResizeJpegO | |||||
| Status SoftDvppDecodeRandomCropResizeJpegOperation::ValidateParams() { | Status SoftDvppDecodeRandomCropResizeJpegOperation::ValidateParams() { | ||||
| // size | // size | ||||
| RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeRandomCropResizeJpeg", size_)); | RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeRandomCropResizeJpeg", size_)); | ||||
| for (int32_t i = 0; i < size_.size(); i++) { | |||||
| if (size_[i] % 2 == 1) { | |||||
| std::string err_msg = "SoftDvppDecodeRandomCropResizeJpeg: size[" + std::to_string(i) + | |||||
| "] must be even values, got: " + std::to_string(size_[i]); | |||||
| MS_LOG(ERROR) << err_msg; | |||||
| RETURN_STATUS_SYNTAX_ERROR(err_msg); | |||||
| } | |||||
| } | |||||
| // scale | // scale | ||||
| RETURN_IF_NOT_OK(ValidateVectorScale("SoftDvppDecodeRandomCropResizeJpeg", scale_)); | RETURN_IF_NOT_OK(ValidateVectorScale("SoftDvppDecodeRandomCropResizeJpeg", scale_)); | ||||
| // ratio | // ratio | ||||
| @@ -1554,6 +1562,14 @@ SoftDvppDecodeResizeJpegOperation::SoftDvppDecodeResizeJpegOperation(std::vector | |||||
| Status SoftDvppDecodeResizeJpegOperation::ValidateParams() { | Status SoftDvppDecodeResizeJpegOperation::ValidateParams() { | ||||
| RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeResizeJpeg", size_)); | RETURN_IF_NOT_OK(ValidateVectorSize("SoftDvppDecodeResizeJpeg", size_)); | ||||
| for (int32_t i = 0; i < size_.size(); i++) { | |||||
| if (size_[i] % 2 == 1) { | |||||
| std::string err_msg = "SoftDvppDecodeResizeJpeg: size[" + std::to_string(i) + | |||||
| "] must be even values, got: " + std::to_string(size_[i]); | |||||
| MS_LOG(ERROR) << err_msg; | |||||
| RETURN_STATUS_SYNTAX_ERROR(err_msg); | |||||
| } | |||||
| } | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -25,6 +25,7 @@ import mindspore._c_dataengine as cde | |||||
| import mindspore.dataset as ds | import mindspore.dataset as ds | ||||
| from ..core import validator_helpers as validator | from ..core import validator_helpers as validator | ||||
| def select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id): | def select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id): | ||||
| """ | """ | ||||
| Create sampler based on user input. | Create sampler based on user input. | ||||
| @@ -615,8 +616,8 @@ class SubsetSampler(BuiltinSampler): | |||||
| Examples: | Examples: | ||||
| >>> indices = [0, 1, 2, 3, 4, 5] | >>> indices = [0, 1, 2, 3, 4, 5] | ||||
| >>> | >>> | ||||
| >>> # creates a SubsetRandomSampler, will sample from the provided indices | |||||
| >>> sampler = ds.SubsetRandomSampler(indices) | |||||
| >>> # creates a SubsetSampler, will sample from the provided indices | |||||
| >>> sampler = ds.SubsetSampler(indices) | |||||
| >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, | >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, | ||||
| ... num_parallel_workers=8, | ... num_parallel_workers=8, | ||||
| ... sampler=sampler) | ... sampler=sampler) | ||||
| @@ -694,7 +695,7 @@ class SubsetRandomSampler(SubsetSampler): | |||||
| Samples the elements randomly from a sequence of indices. | Samples the elements randomly from a sequence of indices. | ||||
| Args: | Args: | ||||
| indices (list[int]): A sequence of indices. | |||||
| indices (Any iterable python object but string): A sequence of indices. | |||||
| num_samples (int, optional): Number of elements to sample (default=None, all elements). | num_samples (int, optional): Number of elements to sample (default=None, all elements). | ||||
| Examples: | Examples: | ||||
| @@ -740,17 +741,16 @@ class IterSampler(Sampler): | |||||
| num_samples (int, optional): Number of elements to sample (default=None, all elements). | num_samples (int, optional): Number of elements to sample (default=None, all elements). | ||||
| Examples: | Examples: | ||||
| >>> class MySampler(): | |||||
| >>> def __iter__(self): | |||||
| >>> for i in range(99, -1, -1): | |||||
| >>> yield i | |||||
| >>> class MySampler: | |||||
| ... def __iter__(self): | |||||
| ... for i in range(99, -1, -1): | |||||
| ... yield i | |||||
| >>> # creates an IterSampler | >>> # creates an IterSampler | ||||
| >>> sampler = ds.IterSampler(sampler=MySampler()) | >>> sampler = ds.IterSampler(sampler=MySampler()) | ||||
| >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, | >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, | ||||
| ... num_parallel_workers=8, | ... num_parallel_workers=8, | ||||
| ... sampler=sampler) | ... sampler=sampler) | ||||
| """ | """ | ||||
| def __init__(self, sampler, num_samples=None): | def __init__(self, sampler, num_samples=None): | ||||