!9525 [Data]Updated add rotate and orientation to lite

From: @xulei2020
Reviewed-by: @HilbertDavid
Signed-off-by:

mindspore/ccsrc/minddata/dataset/api/vision.cc

@@ -355,6 +355,15 @@ std::shared_ptr<ResizeOperation> Resize(std::vector<int32_t> size, Interpolation
   return op->ValidateParams() ? op : nullptr;
 }
 
+#ifdef ENABLE_ANDROID
+// Function to create RotateOperation.
+std::shared_ptr<RotateOperation> Rotate() {
+  auto op = std::make_shared<RotateOperation>();
+  // Input validation
+  return op->ValidateParams() ? op : nullptr;
+}
+#endif
+
 #ifndef ENABLE_ANDROID
 // Function to create ResizeWithBBoxOperation.
 std::shared_ptr<ResizeWithBBoxOperation> ResizeWithBBox(std::vector<int32_t> size, InterpolationMode interpolation) {
@@ -1818,6 +1827,17 @@ std::shared_ptr<TensorOp> ResizeOperation::Build() {
   return std::make_shared<ResizeOp>(height, width, interpolation_);
 }
 
+#ifdef ENABLE_ANDROID
+// RotateOperation
+RotateOperation::RotateOperation() { rotate_op = std::make_shared<RotateOp>(0); }
+
+Status RotateOperation::ValidateParams() { return Status::OK(); }
+
+std::shared_ptr<TensorOp> RotateOperation::Build() { return rotate_op; }
+
+void RotateOperation::setAngle(uint64_t angle_id) { rotate_op->setAngle(angle_id); }
+#endif
+
 #ifndef ENABLE_ANDROID
 // ResizeWithBBoxOperation
 ResizeWithBBoxOperation::ResizeWithBBoxOperation(std::vector<int32_t> size, InterpolationMode interpolation)

mindspore/ccsrc/minddata/dataset/include/vision.h

@@ -27,7 +27,6 @@
 #include "minddata/dataset/include/transforms.h"
 #include "minddata/dataset/include/vision_lite.h"
 #include "minddata/dataset/util/status.h"
-
 namespace mindspore {
 namespace dataset {
 

mindspore/ccsrc/minddata/dataset/include/vision_lite.h

@@ -26,6 +26,10 @@
 #include "minddata/dataset/include/transforms.h"
 #include "minddata/dataset/util/status.h"
 
+#ifdef ENABLE_ANDROID
+#include "minddata/dataset/kernels/image/rotate_op.h"
+#endif
+
 namespace mindspore {
 namespace dataset {
 
@@ -39,6 +43,9 @@ constexpr char kDecodeOperation[] = "Decode";
 constexpr char kNormalizeOperation[] = "Normalize";
 constexpr char kResizeOperation[] = "Resize";
 
+#ifdef ENABLE_ANDROID
+constexpr char kRotateOperation[] = "Rotate";
+#endif
 // Transform Op classes (in alphabetical order)
 class CenterCropOperation;
 class CropOperation;
@@ -46,6 +53,10 @@ class DecodeOperation;
 class NormalizeOperation;
 class ResizeOperation;
 
+#ifdef ENABLE_ANDROID
+class RotateOperation;
+#endif
+
 /// \brief Function to create a CenterCrop TensorOperation.
 /// \notes Crops the input image at the center to the given size.
 /// \param[in] size A vector representing the output size of the cropped image.
@@ -85,6 +96,12 @@ std::shared_ptr<NormalizeOperation> Normalize(std::vector<float> mean, std::vect
 /// \return Shared pointer to the current TensorOperation.
 std::shared_ptr<ResizeOperation> Resize(std::vector<int32_t> size,
                                         InterpolationMode interpolation = InterpolationMode::kLinear);
+#ifdef ENABLE_ANDROID
+/// \brief Applies an rotate transformation to an image.
+/// \notes Rotate the input image using a specified angle id.
+/// \return Shared pointer to the current TensorOperation.
+std::shared_ptr<RotateOperation> Rotate();
+#endif
 
 class CenterCropOperation : public TensorOperation {
  public:
@@ -168,6 +185,26 @@ class ResizeOperation : public TensorOperation {
   std::vector<int32_t> size_;
   InterpolationMode interpolation_;
 };
+
+#ifdef ENABLE_ANDROID
+class RotateOperation : public TensorOperation {
+ public:
+  RotateOperation();
+
+  ~RotateOperation() = default;
+
+  std::shared_ptr<TensorOp> Build() override;
+
+  Status ValidateParams() override;
+
+  std::string Name() const override { return kRotateOperation; }
+
+  void setAngle(uint64_t angle_id);
+
+ private:
+  std::shared_ptr<RotateOp> rotate_op;
+};
+#endif
 }  // namespace vision
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.cc

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "minddata/dataset/kernels/image/rotate_op.h"
+#include "minddata/dataset/kernels/image/lite_image_utils.h"
+
+namespace mindspore {
+namespace dataset {
+
+RotateOp::RotateOp(int angle_id) : angle_id_(angle_id) {}
+
+Status RotateOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
+  IO_CHECK(input, output);
+  CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "The shape size " + std::to_string(input->shape().Size()) +
+                                                             " of input tensor is invalid");
+  Rotate(input, output, angle_id_);
+  return Status::OK();
+}
+
+}  // namespace dataset
+}  // namespace mindspore

mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.h

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2019 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_ROTATE_OP_H_
+#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_ROTATE_OP_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "minddata/dataset/core/tensor.h"
+#include "minddata/dataset/kernels/tensor_op.h"
+#include "minddata/dataset/util/status.h"
+
+namespace mindspore {
+namespace dataset {
+class RotateOp : public TensorOp {
+ public:
+  /// Constructor
+  explicit RotateOp(int angle_id);
+
+  ~RotateOp() override = default;
+
+  std::string Name() const override { return kRotateOp; }
+
+  Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
+
+  void setAngle(uint64_t angle_id) { angle_id_ = angle_id; }
+
+  /// Member variables
+ protected:
+  std::string kRotateOp = "RotateOp";
+  uint64_t angle_id_;
+};
+
+}  // namespace dataset
+}  // namespace mindspore
+
+#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_ROTATE_OP_H_

mindspore/lite/minddata/CMakeLists.txt

@@ -336,6 +336,7 @@ elseif (BUILD_MINDDATA STREQUAL "wrapper")
             ${MINDDATA_DIR}/kernels/image/crop_op.cc
             ${MINDDATA_DIR}/kernels/image/normalize_op.cc
             ${MINDDATA_DIR}/kernels/image/resize_op.cc
+            ${MINDDATA_DIR}/kernels/image/rotate_op.cc
             ${MINDDATA_DIR}/kernels/data/compose_op.cc
             ${MINDDATA_DIR}/kernels/data/duplicate_op.cc
             ${MINDDATA_DIR}/kernels/data/one_hot_op.cc

mindspore/lite/minddata/wrapper/MDToDApi.cc

@@ -37,6 +37,7 @@ using mindspore::dataset::Path;
 using mindspore::dataset::Tensor;
 
 using TensorOperation = mindspore::dataset::TensorOperation;
+using RotateOperation = mindspore::dataset::vision::RotateOperation;
 
 using mindspore::LogStream;
 using mindspore::MsLogLevel::DEBUG;
@@ -114,18 +115,21 @@ extern "C" MDToDApi *MDToDApi_createPipeLine(MDToDConf_t MDConf) {
     MS_LOG(WARNING) << "MEAN: { " << MDConf.MEAN[0] << ", " << MDConf.MEAN[1] << ", " << MDConf.MEAN[2] << " }";
     MS_LOG(WARNING) << "STD: { " << MDConf.STD[0] << ", " << MDConf.STD[1] << ", " << MDConf.STD[2] << " }";
 
+    MDConf.ResizeSizeWH[0] = 224;
+    MDConf.ResizeSizeWH[1] = 224;
     if ((MDConf.ResizeSizeWH[0] != 0) && (MDConf.ResizeSizeWH[1] != 0)) {
-      std::vector<int> Resize(MDConf.ResizeSizeWH, MDConf.ResizeSizeWH + 2);
-      std::shared_ptr<TensorOperation> resize_op = mindspore::dataset::vision::Resize(Resize);
-      assert(resize_op != nullptr);
+      std::shared_ptr<TensorOperation> resize_op =
+        mindspore::dataset::vision::Resize({MDConf.ResizeSizeWH[0], MDConf.ResizeSizeWH[1]});
       MS_LOG(WARNING) << "Push back resize";
       mapOperations.push_back(resize_op);
+      std::shared_ptr<TensorOperation> rotate_op = mindspore::dataset::vision::Rotate();
+      MS_LOG(WARNING) << "Push back rotate";
+      mapOperations.push_back(rotate_op);
       // hasBatch = true;  Batch not currently supported inMInddata-Lite
     }
     if ((MDConf.CropSizeWH[0] != 0) && (MDConf.CropSizeWH[1] != 0)) {
       std::vector<int> Crop(MDConf.CropSizeWH, MDConf.CropSizeWH + 2);
       std::shared_ptr<TensorOperation> center_crop_op = mindspore::dataset::vision::CenterCrop(Crop);
-      assert(center_crop_op != nullptr);
       MS_LOG(WARNING) << "Push back crop";
       mapOperations.push_back(center_crop_op);
       // hasBatch = true;  Batch not currently supported inMInddata-Lite
@@ -137,9 +141,10 @@ extern "C" MDToDApi *MDToDApi_createPipeLine(MDToDConf_t MDConf) {
   const std::set<std::string> exts = {};
   if (MDConf.fileid > -1) {
     // read specific image using SequentialSampler witn
-    iter = std::make_shared<mindspore::dataset::AlbumOp>(folder_path, true, schema_file, exts, MDConf.fileid);
+    iter =
+      std::make_shared<mindspore::dataset::AlbumOp>(folder_path, true, schema_file, column_names, exts, MDConf.fileid);
   } else {
-    iter = std::make_shared<mindspore::dataset::AlbumOp>(folder_path, true, schema_file, exts);
+    iter = std::make_shared<mindspore::dataset::AlbumOp>(folder_path, true, schema_file, column_names, exts);
   }
 
   // Create objects for the tensor ops
@@ -246,9 +251,9 @@ void GetTensorToBuff(std::unordered_map<std::string, std::shared_ptr<Tensor>> ro
 
 extern "C" int MDToDApi_GetNext(MDToDApi *pMDToDApi, MDToDResult_t *results) {
   MS_LOG(INFO) << "Start GetNext";
-  if (pMDToDApi == nullptr) {
+  if (pMDToDApi == nullptr || pMDToDApi->_iter == nullptr) {
     MS_LOG(ERROR) << "GetNext called with null ptr. abort";
-    assert(pMDToDApi != nullptr);
+    return -1;
   }
 
   // Set defualt
@@ -268,12 +273,22 @@ extern "C" int MDToDApi_GetNext(MDToDApi *pMDToDApi, MDToDResult_t *results) {
   if (row.size() != 0 && ret) {
     if ((pMDToDApi->_augs).size() > 0) {
       // String and Tensors
-      GetTensorToBuff(row, "image_filename", pMDToDApi->_hasBatch, &results->fileNameBuff);
+      uint32_t orientation;
+      row["orientation"]->GetItemAt(&orientation, {});
+      MS_LOG(WARNING) << "get orientation from row = " << orientation;
       // for each operation, run eager mode, single threaded operation, will have to memcpy
       // regardless
       for (int i = 0; i < (pMDToDApi->_augs).size(); i++) {
         // each Execute call will invoke a memcpy, this cannot really be optimized further
         // for this use case, std move is added for fail save.
+        if (pMDToDApi->_augs[i]->Name() == "Rotate") {
+          if (orientation > 1) {
+            RotateOperation *p = static_cast<RotateOperation *>(pMDToDApi->_augs[i].get());
+            p->setAngle(orientation);
+          } else {
+            continue;
+          }
+        }
         row["image"] = mindspore::dataset::Execute((pMDToDApi->_augs)[i])(std::move(row["image"]));
         if (row["image"] == nullptr) {
           // nullptr means that the eager mode image processing failed, we fail in this case

mindspore/lite/minddata/wrapper/album_op_android.cc

@@ -24,7 +24,7 @@ namespace mindspore {
 namespace dataset {
 
 AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
-                 const std::set<std::string> &exts)
+                 const std::vector<std::string> &column_names, const std::set<std::string> &exts)
     : folder_path_(file_dir),
       decode_(do_decode),
       extensions_(exts),
@@ -35,12 +35,13 @@ AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string
       dirname_offset_(0),
       sampler_(false),
       sampler_index_(0),
-      rotate_(true) {
+      rotate_(true),
+      column_names_(column_names) {
   PrescanEntry();
 }
 
 AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
-                 const std::set<std::string> &exts, uint32_t index)
+                 const std::vector<std::string> &column_names, const std::set<std::string> &exts, uint32_t index)
     : folder_path_(file_dir),
       decode_(do_decode),
       extensions_(exts),
@@ -51,7 +52,8 @@ AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string
       dirname_offset_(0),
       sampler_(true),
       sampler_index_(index),
-      rotate_(true) {
+      rotate_(true),
+      column_names_(column_names) {
   PrescanEntry();
 }
 
@@ -78,10 +80,6 @@ Status AlbumOp::PrescanEntry() {
     data_schema_->LoadSchemaFile(schema_file_, columns_to_load_);
   }
 
-  for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) {
-    column_name_id_map_[data_schema_->column(i).name()] = i;
-  }
-
   Path folder(folder_path_);
   dirname_offset_ = folder_path_.length();
   std::shared_ptr<Path::DirIterator> dirItr = Path::DirIterator::OpenDirectory(&folder);
@@ -169,6 +167,15 @@ bool AlbumOp::CheckImageType(const std::string &file_name, bool *valid) {
   return true;
 }
 
+bool AlbumOp::IsReadColumn(const std::string &column_name) {
+  for (uint32_t i = 0; i < this->column_names_.size(); i++) {
+    if (this->column_names_[i] == column_name) {
+      return true;
+    }
+  }
+  return false;
+}
+
 Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col_num, TensorPtr *tensor) {
   TensorPtr image;
   TensorPtr rotate_tensor;
@@ -210,25 +217,10 @@ Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col
   RETURN_IF_NOT_OK(Tensor::CreateFromFile(image_file_path, &image));
   Status rc;
   if (decode_ && valid) {
-    int orientation = GetOrientation(image_file_path);
-    if (orientation > 1 && this->rotate_) {
-      rc = Decode(image, &rotate_tensor);
-      if (rc.IsError()) {
-        RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
-        return Status::OK();
-      }
-
-      rc = Rotate(rotate_tensor, tensor, orientation);
-      if (rc.IsError()) {
-        RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
-        return Status::OK();
-      }
-    } else {
-      rc = Decode(image, tensor);
-      if (rc.IsError()) {
-        RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
-        return Status::OK();
-      }
+    rc = Decode(image, tensor);
+    if (rc.IsError()) {
+      RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
+      return Status::OK();
     }
   }
   return Status::OK();
@@ -244,6 +236,10 @@ int AlbumOp::GetOrientation(const std::string &folder_path) {
   fseek(fp, 0, SEEK_END);
   int64_t fsize = ftell(fp);
   rewind(fp);
+  if (fsize > INT_MAX) {
+    fclose(fp);
+    return 0;
+  }
   unsigned char *buf = new unsigned char[fsize];
   if (fread(buf, 1, fsize, fp) != fsize) {
     MS_LOG(WARNING) << "read file size error for EXIF:  file = " << folder_path;
@@ -257,10 +253,7 @@ int AlbumOp::GetOrientation(const std::string &folder_path) {
   mindspore::dataset::ExifInfo result;
   int code = result.parseOrientation(buf, fsize);
   delete[] buf;
-  if (code == 0) {
-    MS_LOG(WARNING) << "Error parsing EXIF, use default code = " << code << ".";
-  }
-
+  MS_LOG(WARNING) << "AlbumOp::GetOrientation:  orientation= " << code << ".";
   return code;
 }
 
@@ -429,6 +422,9 @@ Status AlbumOp::LoadTensorRow(row_id_type row_id, const std::string &file,
 
       // loop over each column descriptor, this can optimized by switch cases
       for (int32_t i = 0; i < columns; i++) {
+        if (!IsReadColumn(data_schema_->column(i).name())) {
+          continue;
+        }
         // special case to handle
         if (data_schema_->column(i).name() == "id") {
           // id is internal, special case to load from file
@@ -469,6 +465,10 @@ Status AlbumOp::LoadTensorRow(row_id_type row_id, const std::string &file,
           TensorPtr tensor;
           RETURN_IF_NOT_OK(LoadImageTensor(image_file_path, i, &tensor));
           (*map_row)[data_schema_->column(i).name()] = tensor;
+          uint32_t orientation = GetOrientation(image_file_path);
+          TensorPtr scalar_tensor;
+          RETURN_IF_NOT_OK(Tensor::CreateScalar<uint32_t>(orientation, &scalar_tensor));
+          (*map_row)["orientation"] = scalar_tensor;
           continue;
         }
         // load float value

mindspore/lite/minddata/wrapper/album_op_android.h

@@ -51,7 +51,7 @@ class AlbumOp {
   /// \param[in] exts - set of file extensions to read, if empty, read everything under the dir
   /// \param[in] rotate - rotate image exif orientation
   AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
-          const std::set<std::string> &exts);
+          const std::vector<std::string> &column_names, const std::set<std::string> &exts);
 
   /// \brief Constructor
   /// \param[in] file_dir - directory of Album
@@ -61,7 +61,7 @@ class AlbumOp {
   /// \param[in] index - the specific file index
   /// \param[in] rotate - rotate image exif orientation
   AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
-          const std::set<std::string> &exts, uint32_t index);
+          const std::vector<std::string> &column_names, const std::set<std::string> &exts, uint32_t index);
 
   /// \brief Destructor.
   ~AlbumOp() = default;
@@ -162,6 +162,7 @@ class AlbumOp {
   /// \param[in] file file path
   int GetOrientation(const std::string &file);
 
+  bool IsReadColumn(const std::string &column_name);
   std::string folder_path_;  // directory of image folder
   bool decode_;
   std::vector<std::string> columns_to_load_;
@@ -175,8 +176,8 @@ class AlbumOp {
   bool sampler_;
   int64_t sampler_index_;
   std::vector<std::string> image_rows_;
-  std::unordered_map<std::string, int32_t> column_name_id_map_;
   bool rotate_;
+  std::vector<std::string> column_names_;
 };
 }  // namespace dataset
 }  // namespace mindspore