diff --git a/mindspore/ccsrc/minddata/dataset/api/vision.cc b/mindspore/ccsrc/minddata/dataset/api/vision.cc index e68abeaf7b..13e40143f7 100644 --- a/mindspore/ccsrc/minddata/dataset/api/vision.cc +++ b/mindspore/ccsrc/minddata/dataset/api/vision.cc @@ -355,6 +355,15 @@ std::shared_ptr Resize(std::vector size, Interpolation return op->ValidateParams() ? op : nullptr; } +#ifdef ENABLE_ANDROID +// Function to create RotateOperation. +std::shared_ptr Rotate() { + auto op = std::make_shared(); + // Input validation + return op->ValidateParams() ? op : nullptr; +} +#endif + #ifndef ENABLE_ANDROID // Function to create ResizeWithBBoxOperation. std::shared_ptr ResizeWithBBox(std::vector size, InterpolationMode interpolation) { @@ -1818,6 +1827,17 @@ std::shared_ptr ResizeOperation::Build() { return std::make_shared(height, width, interpolation_); } +#ifdef ENABLE_ANDROID +// RotateOperation +RotateOperation::RotateOperation() { rotate_op = std::make_shared(0); } + +Status RotateOperation::ValidateParams() { return Status::OK(); } + +std::shared_ptr RotateOperation::Build() { return rotate_op; } + +void RotateOperation::setAngle(uint64_t angle_id) { rotate_op->setAngle(angle_id); } +#endif + #ifndef ENABLE_ANDROID // ResizeWithBBoxOperation ResizeWithBBoxOperation::ResizeWithBBoxOperation(std::vector size, InterpolationMode interpolation) diff --git a/mindspore/ccsrc/minddata/dataset/include/vision.h b/mindspore/ccsrc/minddata/dataset/include/vision.h index ce255743cc..edc5977623 100644 --- a/mindspore/ccsrc/minddata/dataset/include/vision.h +++ b/mindspore/ccsrc/minddata/dataset/include/vision.h @@ -27,7 +27,6 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/vision_lite.h" #include "minddata/dataset/util/status.h" - namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/include/vision_lite.h b/mindspore/ccsrc/minddata/dataset/include/vision_lite.h index 6310380736..e85f0cf30d 100644 --- a/mindspore/ccsrc/minddata/dataset/include/vision_lite.h +++ b/mindspore/ccsrc/minddata/dataset/include/vision_lite.h @@ -26,6 +26,10 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/util/status.h" +#ifdef ENABLE_ANDROID +#include "minddata/dataset/kernels/image/rotate_op.h" +#endif + namespace mindspore { namespace dataset { @@ -39,6 +43,9 @@ constexpr char kDecodeOperation[] = "Decode"; constexpr char kNormalizeOperation[] = "Normalize"; constexpr char kResizeOperation[] = "Resize"; +#ifdef ENABLE_ANDROID +constexpr char kRotateOperation[] = "Rotate"; +#endif // Transform Op classes (in alphabetical order) class CenterCropOperation; class CropOperation; @@ -46,6 +53,10 @@ class DecodeOperation; class NormalizeOperation; class ResizeOperation; +#ifdef ENABLE_ANDROID +class RotateOperation; +#endif + /// \brief Function to create a CenterCrop TensorOperation. /// \notes Crops the input image at the center to the given size. /// \param[in] size A vector representing the output size of the cropped image. @@ -85,6 +96,12 @@ std::shared_ptr Normalize(std::vector mean, std::vect /// \return Shared pointer to the current TensorOperation. std::shared_ptr Resize(std::vector size, InterpolationMode interpolation = InterpolationMode::kLinear); +#ifdef ENABLE_ANDROID +/// \brief Applies an rotate transformation to an image. +/// \notes Rotate the input image using a specified angle id. +/// \return Shared pointer to the current TensorOperation. +std::shared_ptr Rotate(); +#endif class CenterCropOperation : public TensorOperation { public: @@ -168,6 +185,26 @@ class ResizeOperation : public TensorOperation { std::vector size_; InterpolationMode interpolation_; }; + +#ifdef ENABLE_ANDROID +class RotateOperation : public TensorOperation { + public: + RotateOperation(); + + ~RotateOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + std::string Name() const override { return kRotateOperation; } + + void setAngle(uint64_t angle_id); + + private: + std::shared_ptr rotate_op; +}; +#endif } // namespace vision } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.cc new file mode 100644 index 0000000000..c5eb281ade --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/kernels/image/rotate_op.h" +#include "minddata/dataset/kernels/image/lite_image_utils.h" + +namespace mindspore { +namespace dataset { + +RotateOp::RotateOp(int angle_id) : angle_id_(angle_id) {} + +Status RotateOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "The shape size " + std::to_string(input->shape().Size()) + + " of input tensor is invalid"); + Rotate(input, output, angle_id_); + return Status::OK(); +} + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.h b/mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.h new file mode 100644 index 0000000000..2cca502fe8 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_ROTATE_OP_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_ROTATE_OP_H_ + +#include +#include +#include + +#include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/kernels/tensor_op.h" +#include "minddata/dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RotateOp : public TensorOp { + public: + /// Constructor + explicit RotateOp(int angle_id); + + ~RotateOp() override = default; + + std::string Name() const override { return kRotateOp; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + void setAngle(uint64_t angle_id) { angle_id_ = angle_id; } + + /// Member variables + protected: + std::string kRotateOp = "RotateOp"; + uint64_t angle_id_; +}; + +} // namespace dataset +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_ROTATE_OP_H_ diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index 1908466178..a3f34fcfaa 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -336,6 +336,7 @@ elseif (BUILD_MINDDATA STREQUAL "wrapper") ${MINDDATA_DIR}/kernels/image/crop_op.cc ${MINDDATA_DIR}/kernels/image/normalize_op.cc ${MINDDATA_DIR}/kernels/image/resize_op.cc + ${MINDDATA_DIR}/kernels/image/rotate_op.cc ${MINDDATA_DIR}/kernels/data/compose_op.cc ${MINDDATA_DIR}/kernels/data/duplicate_op.cc ${MINDDATA_DIR}/kernels/data/one_hot_op.cc diff --git a/mindspore/lite/minddata/wrapper/MDToDApi.cc b/mindspore/lite/minddata/wrapper/MDToDApi.cc index 281ff4ca24..55ccbf92aa 100644 --- a/mindspore/lite/minddata/wrapper/MDToDApi.cc +++ b/mindspore/lite/minddata/wrapper/MDToDApi.cc @@ -37,6 +37,7 @@ using mindspore::dataset::Path; using mindspore::dataset::Tensor; using TensorOperation = mindspore::dataset::TensorOperation; +using RotateOperation = mindspore::dataset::vision::RotateOperation; using mindspore::LogStream; using mindspore::MsLogLevel::DEBUG; @@ -114,18 +115,21 @@ extern "C" MDToDApi *MDToDApi_createPipeLine(MDToDConf_t MDConf) { MS_LOG(WARNING) << "MEAN: { " << MDConf.MEAN[0] << ", " << MDConf.MEAN[1] << ", " << MDConf.MEAN[2] << " }"; MS_LOG(WARNING) << "STD: { " << MDConf.STD[0] << ", " << MDConf.STD[1] << ", " << MDConf.STD[2] << " }"; + MDConf.ResizeSizeWH[0] = 224; + MDConf.ResizeSizeWH[1] = 224; if ((MDConf.ResizeSizeWH[0] != 0) && (MDConf.ResizeSizeWH[1] != 0)) { - std::vector Resize(MDConf.ResizeSizeWH, MDConf.ResizeSizeWH + 2); - std::shared_ptr resize_op = mindspore::dataset::vision::Resize(Resize); - assert(resize_op != nullptr); + std::shared_ptr resize_op = + mindspore::dataset::vision::Resize({MDConf.ResizeSizeWH[0], MDConf.ResizeSizeWH[1]}); MS_LOG(WARNING) << "Push back resize"; mapOperations.push_back(resize_op); + std::shared_ptr rotate_op = mindspore::dataset::vision::Rotate(); + MS_LOG(WARNING) << "Push back rotate"; + mapOperations.push_back(rotate_op); // hasBatch = true; Batch not currently supported inMInddata-Lite } if ((MDConf.CropSizeWH[0] != 0) && (MDConf.CropSizeWH[1] != 0)) { std::vector Crop(MDConf.CropSizeWH, MDConf.CropSizeWH + 2); std::shared_ptr center_crop_op = mindspore::dataset::vision::CenterCrop(Crop); - assert(center_crop_op != nullptr); MS_LOG(WARNING) << "Push back crop"; mapOperations.push_back(center_crop_op); // hasBatch = true; Batch not currently supported inMInddata-Lite @@ -137,9 +141,10 @@ extern "C" MDToDApi *MDToDApi_createPipeLine(MDToDConf_t MDConf) { const std::set exts = {}; if (MDConf.fileid > -1) { // read specific image using SequentialSampler witn - iter = std::make_shared(folder_path, true, schema_file, exts, MDConf.fileid); + iter = + std::make_shared(folder_path, true, schema_file, column_names, exts, MDConf.fileid); } else { - iter = std::make_shared(folder_path, true, schema_file, exts); + iter = std::make_shared(folder_path, true, schema_file, column_names, exts); } // Create objects for the tensor ops @@ -246,9 +251,9 @@ void GetTensorToBuff(std::unordered_map> ro extern "C" int MDToDApi_GetNext(MDToDApi *pMDToDApi, MDToDResult_t *results) { MS_LOG(INFO) << "Start GetNext"; - if (pMDToDApi == nullptr) { + if (pMDToDApi == nullptr || pMDToDApi->_iter == nullptr) { MS_LOG(ERROR) << "GetNext called with null ptr. abort"; - assert(pMDToDApi != nullptr); + return -1; } // Set defualt @@ -268,12 +273,22 @@ extern "C" int MDToDApi_GetNext(MDToDApi *pMDToDApi, MDToDResult_t *results) { if (row.size() != 0 && ret) { if ((pMDToDApi->_augs).size() > 0) { // String and Tensors - GetTensorToBuff(row, "image_filename", pMDToDApi->_hasBatch, &results->fileNameBuff); + uint32_t orientation; + row["orientation"]->GetItemAt(&orientation, {}); + MS_LOG(WARNING) << "get orientation from row = " << orientation; // for each operation, run eager mode, single threaded operation, will have to memcpy // regardless for (int i = 0; i < (pMDToDApi->_augs).size(); i++) { // each Execute call will invoke a memcpy, this cannot really be optimized further // for this use case, std move is added for fail save. + if (pMDToDApi->_augs[i]->Name() == "Rotate") { + if (orientation > 1) { + RotateOperation *p = static_cast(pMDToDApi->_augs[i].get()); + p->setAngle(orientation); + } else { + continue; + } + } row["image"] = mindspore::dataset::Execute((pMDToDApi->_augs)[i])(std::move(row["image"])); if (row["image"] == nullptr) { // nullptr means that the eager mode image processing failed, we fail in this case diff --git a/mindspore/lite/minddata/wrapper/album_op_android.cc b/mindspore/lite/minddata/wrapper/album_op_android.cc index a35b2b0d63..52f47dc5e8 100644 --- a/mindspore/lite/minddata/wrapper/album_op_android.cc +++ b/mindspore/lite/minddata/wrapper/album_op_android.cc @@ -24,7 +24,7 @@ namespace mindspore { namespace dataset { AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file, - const std::set &exts) + const std::vector &column_names, const std::set &exts) : folder_path_(file_dir), decode_(do_decode), extensions_(exts), @@ -35,12 +35,13 @@ AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string dirname_offset_(0), sampler_(false), sampler_index_(0), - rotate_(true) { + rotate_(true), + column_names_(column_names) { PrescanEntry(); } AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file, - const std::set &exts, uint32_t index) + const std::vector &column_names, const std::set &exts, uint32_t index) : folder_path_(file_dir), decode_(do_decode), extensions_(exts), @@ -51,7 +52,8 @@ AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string dirname_offset_(0), sampler_(true), sampler_index_(index), - rotate_(true) { + rotate_(true), + column_names_(column_names) { PrescanEntry(); } @@ -78,10 +80,6 @@ Status AlbumOp::PrescanEntry() { data_schema_->LoadSchemaFile(schema_file_, columns_to_load_); } - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } - Path folder(folder_path_); dirname_offset_ = folder_path_.length(); std::shared_ptr dirItr = Path::DirIterator::OpenDirectory(&folder); @@ -169,6 +167,15 @@ bool AlbumOp::CheckImageType(const std::string &file_name, bool *valid) { return true; } +bool AlbumOp::IsReadColumn(const std::string &column_name) { + for (uint32_t i = 0; i < this->column_names_.size(); i++) { + if (this->column_names_[i] == column_name) { + return true; + } + } + return false; +} + Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col_num, TensorPtr *tensor) { TensorPtr image; TensorPtr rotate_tensor; @@ -210,25 +217,10 @@ Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col RETURN_IF_NOT_OK(Tensor::CreateFromFile(image_file_path, &image)); Status rc; if (decode_ && valid) { - int orientation = GetOrientation(image_file_path); - if (orientation > 1 && this->rotate_) { - rc = Decode(image, &rotate_tensor); - if (rc.IsError()) { - RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor)); - return Status::OK(); - } - - rc = Rotate(rotate_tensor, tensor, orientation); - if (rc.IsError()) { - RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor)); - return Status::OK(); - } - } else { - rc = Decode(image, tensor); - if (rc.IsError()) { - RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor)); - return Status::OK(); - } + rc = Decode(image, tensor); + if (rc.IsError()) { + RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor)); + return Status::OK(); } } return Status::OK(); @@ -244,6 +236,10 @@ int AlbumOp::GetOrientation(const std::string &folder_path) { fseek(fp, 0, SEEK_END); int64_t fsize = ftell(fp); rewind(fp); + if (fsize > INT_MAX) { + fclose(fp); + return 0; + } unsigned char *buf = new unsigned char[fsize]; if (fread(buf, 1, fsize, fp) != fsize) { MS_LOG(WARNING) << "read file size error for EXIF: file = " << folder_path; @@ -257,10 +253,7 @@ int AlbumOp::GetOrientation(const std::string &folder_path) { mindspore::dataset::ExifInfo result; int code = result.parseOrientation(buf, fsize); delete[] buf; - if (code == 0) { - MS_LOG(WARNING) << "Error parsing EXIF, use default code = " << code << "."; - } - + MS_LOG(WARNING) << "AlbumOp::GetOrientation: orientation= " << code << "."; return code; } @@ -429,6 +422,9 @@ Status AlbumOp::LoadTensorRow(row_id_type row_id, const std::string &file, // loop over each column descriptor, this can optimized by switch cases for (int32_t i = 0; i < columns; i++) { + if (!IsReadColumn(data_schema_->column(i).name())) { + continue; + } // special case to handle if (data_schema_->column(i).name() == "id") { // id is internal, special case to load from file @@ -469,6 +465,10 @@ Status AlbumOp::LoadTensorRow(row_id_type row_id, const std::string &file, TensorPtr tensor; RETURN_IF_NOT_OK(LoadImageTensor(image_file_path, i, &tensor)); (*map_row)[data_schema_->column(i).name()] = tensor; + uint32_t orientation = GetOrientation(image_file_path); + TensorPtr scalar_tensor; + RETURN_IF_NOT_OK(Tensor::CreateScalar(orientation, &scalar_tensor)); + (*map_row)["orientation"] = scalar_tensor; continue; } // load float value diff --git a/mindspore/lite/minddata/wrapper/album_op_android.h b/mindspore/lite/minddata/wrapper/album_op_android.h index b958eab35e..d9e472252a 100644 --- a/mindspore/lite/minddata/wrapper/album_op_android.h +++ b/mindspore/lite/minddata/wrapper/album_op_android.h @@ -51,7 +51,7 @@ class AlbumOp { /// \param[in] exts - set of file extensions to read, if empty, read everything under the dir /// \param[in] rotate - rotate image exif orientation AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file, - const std::set &exts); + const std::vector &column_names, const std::set &exts); /// \brief Constructor /// \param[in] file_dir - directory of Album @@ -61,7 +61,7 @@ class AlbumOp { /// \param[in] index - the specific file index /// \param[in] rotate - rotate image exif orientation AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file, - const std::set &exts, uint32_t index); + const std::vector &column_names, const std::set &exts, uint32_t index); /// \brief Destructor. ~AlbumOp() = default; @@ -162,6 +162,7 @@ class AlbumOp { /// \param[in] file file path int GetOrientation(const std::string &file); + bool IsReadColumn(const std::string &column_name); std::string folder_path_; // directory of image folder bool decode_; std::vector columns_to_load_; @@ -175,8 +176,8 @@ class AlbumOp { bool sampler_; int64_t sampler_index_; std::vector image_rows_; - std::unordered_map column_name_id_map_; bool rotate_; + std::vector column_names_; }; } // namespace dataset } // namespace mindspore