| @@ -30,6 +30,7 @@ | |||
| #include "dataset/kernels/image/random_crop_and_resize_op.h" | |||
| #include "dataset/kernels/image/random_crop_op.h" | |||
| #include "dataset/kernels/image/random_horizontal_flip_op.h" | |||
| #include "dataset/kernels/image/random_horizontal_flip_bbox_op.h" | |||
| #include "dataset/kernels/image/random_resize_op.h" | |||
| #include "dataset/kernels/image/random_rotation_op.h" | |||
| #include "dataset/kernels/image/random_vertical_flip_op.h" | |||
| @@ -37,6 +38,7 @@ | |||
| #include "dataset/kernels/image/resize_bilinear_op.h" | |||
| #include "dataset/kernels/image/resize_op.h" | |||
| #include "dataset/kernels/image/uniform_aug_op.h" | |||
| #include "dataset/kernels/image/bounding_box_augment_op.h" | |||
| #include "dataset/kernels/data/fill_op.h" | |||
| #include "dataset/kernels/data/slice_op.h" | |||
| #include "dataset/kernels/data/type_cast_op.h" | |||
| @@ -343,6 +345,11 @@ void bindTensorOps1(py::module *m) { | |||
| .def(py::init<std::vector<std::shared_ptr<TensorOp>>, int32_t>(), py::arg("operations"), | |||
| py::arg("NumOps") = UniformAugOp::kDefNumOps); | |||
| (void)py::class_<BoundingBoxAugOp, TensorOp, std::shared_ptr<BoundingBoxAugOp>>( | |||
| *m, "BoundingBoxAugOp", "Tensor operation to apply a transformation on a random choice of bounding boxes.") | |||
| .def(py::init<std::shared_ptr<TensorOp>, float>(), py::arg("transform"), | |||
| py::arg("ratio") = BoundingBoxAugOp::defRatio); | |||
| (void)py::class_<ResizeBilinearOp, TensorOp, std::shared_ptr<ResizeBilinearOp>>( | |||
| *m, "ResizeBilinearOp", | |||
| "Tensor operation to resize an image using " | |||
| @@ -357,6 +364,11 @@ void bindTensorOps1(py::module *m) { | |||
| (void)py::class_<RandomHorizontalFlipOp, TensorOp, std::shared_ptr<RandomHorizontalFlipOp>>( | |||
| *m, "RandomHorizontalFlipOp", "Tensor operation to randomly flip an image horizontally.") | |||
| .def(py::init<float>(), py::arg("probability") = RandomHorizontalFlipOp::kDefProbability); | |||
| (void)py::class_<RandomHorizontalFlipWithBBoxOp, TensorOp, std::shared_ptr<RandomHorizontalFlipWithBBoxOp>>( | |||
| *m, "RandomHorizontalFlipWithBBoxOp", | |||
| "Tensor operation to randomly flip an image horizontally, while flipping bounding boxes.") | |||
| .def(py::init<float>(), py::arg("probability") = RandomHorizontalFlipWithBBoxOp::kDefProbability); | |||
| } | |||
| void bindTensorOps2(py::module *m) { | |||
| @@ -13,6 +13,8 @@ add_library(kernels-image OBJECT | |||
| random_crop_and_resize_op.cc | |||
| random_crop_op.cc | |||
| random_horizontal_flip_op.cc | |||
| random_horizontal_flip_bbox_op.cc | |||
| bounding_box_augment_op.cc | |||
| random_resize_op.cc | |||
| random_rotation_op.cc | |||
| random_vertical_flip_op.cc | |||
| @@ -0,0 +1,77 @@ | |||
| /** | |||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| */ | |||
| #include <vector> | |||
| #include <utility> | |||
| #include "dataset/kernels/image/bounding_box_augment_op.h" | |||
| #include "dataset/kernels/image/resize_op.h" | |||
| #include "dataset/kernels/image/image_utils.h" | |||
| #include "dataset/core/cv_tensor.h" | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| const float BoundingBoxAugOp::defRatio = 0.3; | |||
| BoundingBoxAugOp::BoundingBoxAugOp(std::shared_ptr<TensorOp> transform, float ratio) | |||
| : ratio_(ratio), transform_(std::move(transform)) {} | |||
| Status BoundingBoxAugOp::Compute(const TensorRow &input, TensorRow *output) { | |||
| IO_CHECK_VECTOR(input, output); | |||
| BOUNDING_BOX_CHECK(input); // check if bounding boxes are valid | |||
| uint32_t num_of_boxes = input[1]->shape()[0]; | |||
| uint32_t num_to_aug = num_of_boxes * ratio_; // cast to int | |||
| std::vector<uint32_t> boxes(num_of_boxes); | |||
| std::vector<uint32_t> selected_boxes; | |||
| for (uint32_t i = 0; i < num_of_boxes; i++) boxes[i] = i; | |||
| // sample bboxes according to ratio picked by user | |||
| std::random_device rd; | |||
| std::sample(boxes.begin(), boxes.end(), std::back_inserter(selected_boxes), num_to_aug, std::mt19937(rd())); | |||
| std::shared_ptr<Tensor> crop_out; | |||
| std::shared_ptr<Tensor> res_out; | |||
| std::shared_ptr<CVTensor> input_restore = CVTensor::AsCVTensor(input[0]); | |||
| for (uint32_t i = 0; i < num_to_aug; i++) { | |||
| uint32_t min_x = 0; | |||
| uint32_t min_y = 0; | |||
| uint32_t b_w = 0; | |||
| uint32_t b_h = 0; | |||
| // get the required items | |||
| input[1]->GetItemAt<uint32_t>(&min_x, {selected_boxes[i], 0}); | |||
| input[1]->GetItemAt<uint32_t>(&min_y, {selected_boxes[i], 1}); | |||
| input[1]->GetItemAt<uint32_t>(&b_w, {selected_boxes[i], 2}); | |||
| input[1]->GetItemAt<uint32_t>(&b_h, {selected_boxes[i], 3}); | |||
| Crop(input_restore, &crop_out, min_x, min_y, b_w, b_h); | |||
| // transform the cropped bbox region | |||
| transform_->Compute(crop_out, &res_out); | |||
| // place the transformed region back in the restored input | |||
| std::shared_ptr<CVTensor> res_img = CVTensor::AsCVTensor(res_out); | |||
| // check if transformed crop is out of bounds of the box | |||
| if (res_img->mat().cols > b_w || res_img->mat().rows > b_h || res_img->mat().cols < b_w || | |||
| res_img->mat().rows < b_h) { | |||
| // if so, resize to fit in the box | |||
| std::shared_ptr<TensorOp> resize_op = std::make_shared<ResizeOp>(b_h, b_w); | |||
| resize_op->Compute(std::static_pointer_cast<Tensor>(res_img), &res_out); | |||
| res_img = CVTensor::AsCVTensor(res_out); | |||
| } | |||
| res_img->mat().copyTo(input_restore->mat()(cv::Rect(min_x, min_y, res_img->mat().cols, res_img->mat().rows))); | |||
| } | |||
| (*output).push_back(std::move(std::static_pointer_cast<Tensor>(input_restore))); | |||
| (*output).push_back(input[1]); | |||
| return Status::OK(); | |||
| } | |||
| } // namespace dataset | |||
| } // namespace mindspore | |||
| @@ -0,0 +1,59 @@ | |||
| /** | |||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| */ | |||
| #ifndef DATASET_KERNELS_IMAGE_BOUNDING_BOX_AUGMENT_OP_H_ | |||
| #define DATASET_KERNELS_IMAGE_BOUNDING_BOX_AUGMENT_OP_H_ | |||
| #include <memory> | |||
| #include <random> | |||
| #include <cstdlib> | |||
| #include <opencv2/imgproc/imgproc.hpp> | |||
| #include "dataset/core/tensor.h" | |||
| #include "dataset/kernels/tensor_op.h" | |||
| #include "dataset/util/status.h" | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| class BoundingBoxAugOp : public TensorOp { | |||
| public: | |||
| // Default values, also used by python_bindings.cc | |||
| static const float defRatio; | |||
| // Constructor for BoundingBoxAugmentOp | |||
| // @param std::shared_ptr<TensorOp> transform transform: C++ opration to apply on select bounding boxes | |||
| // @param float ratio: ratio of bounding boxes to have the transform applied on | |||
| BoundingBoxAugOp(std::shared_ptr<TensorOp> transform, float ratio); | |||
| ~BoundingBoxAugOp() override = default; | |||
| // Provide stream operator for displaying it | |||
| friend std::ostream &operator<<(std::ostream &out, const BoundingBoxAugOp &so) { | |||
| so.Print(out); | |||
| return out; | |||
| } | |||
| void Print(std::ostream &out) const override { out << "BoundingBoxAugOp"; } | |||
| Status Compute(const TensorRow &input, TensorRow *output) override; | |||
| private: | |||
| float ratio_; | |||
| std::shared_ptr<TensorOp> transform_; | |||
| }; | |||
| } // namespace dataset | |||
| } // namespace mindspore | |||
| #endif // DATASET_KERNELS_IMAGE_BOUNDING_BOX_AUGMENT_OP_H_ | |||
| @@ -0,0 +1,61 @@ | |||
| /** | |||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| */ | |||
| #include <utility> | |||
| #include "dataset/kernels/image/random_horizontal_flip_bbox_op.h" | |||
| #include "dataset/kernels/image/image_utils.h" | |||
| #include "dataset/util/status.h" | |||
| #include "dataset/core/cv_tensor.h" | |||
| #include "dataset/core/pybind_support.h" | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| const float RandomHorizontalFlipWithBBoxOp::kDefProbability = 0.5; | |||
| Status RandomHorizontalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) { | |||
| IO_CHECK_VECTOR(input, output); | |||
| BOUNDING_BOX_CHECK(input); | |||
| if (distribution_(rnd_)) { | |||
| // To test bounding boxes algorithm, create random bboxes from image dims | |||
| size_t numOfBBoxes = input[1]->shape()[0]; // set to give number of bboxes | |||
| float imgCenter = (input[0]->shape()[1] / 2); // get the center of the image | |||
| for (int i = 0; i < numOfBBoxes; i++) { | |||
| uint32_t b_w = 0; // bounding box width | |||
| uint32_t min_x = 0; | |||
| // get the required items | |||
| input[1]->GetItemAt<uint32_t>(&min_x, {i, 0}); | |||
| input[1]->GetItemAt<uint32_t>(&b_w, {i, 2}); | |||
| // do the flip | |||
| float diff = imgCenter - min_x; // get distance from min_x to center | |||
| uint32_t refl_min_x = diff + imgCenter; // get reflection of min_x | |||
| uint32_t new_min_x = refl_min_x - b_w; // subtract from the reflected min_x to get the new one | |||
| input[1]->SetItemAt<uint32_t>({i, 0}, new_min_x); | |||
| } | |||
| (*output).push_back(nullptr); | |||
| (*output).push_back(nullptr); | |||
| // move input to output pointer of bounding boxes | |||
| (*output)[1] = std::move(input[1]); | |||
| // perform HorizontalFlip on the image | |||
| std::shared_ptr<CVTensor> input_cv = CVTensor::AsCVTensor(std::move(input[0])); | |||
| return HorizontalFlip(std::static_pointer_cast<Tensor>(input_cv), &(*output)[0]); | |||
| } | |||
| *output = input; | |||
| return Status::OK(); | |||
| } | |||
| } // namespace dataset | |||
| } // namespace mindspore | |||
| @@ -0,0 +1,62 @@ | |||
| /** | |||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| */ | |||
| #ifndef DATASET_KERNELS_IMAGE_RANDOM_HORIZONTAL_FLIP_BBOX_OP_H_ | |||
| #define DATASET_KERNELS_IMAGE_RANDOM_HORIZONTAL_FLIP_BBOX_OP_H_ | |||
| #include <pybind11/numpy.h> | |||
| #include <pybind11/stl.h> | |||
| #include <memory> | |||
| #include <random> | |||
| #include <cstdlib> | |||
| #include <opencv2/imgproc/imgproc.hpp> | |||
| #include "dataset/core/tensor.h" | |||
| #include "dataset/kernels/tensor_op.h" | |||
| #include "dataset/util/random.h" | |||
| #include "dataset/util/status.h" | |||
| #include "pybind11/pybind11.h" | |||
| #include "pybind11/stl_bind.h" | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| class RandomHorizontalFlipWithBBoxOp : public TensorOp { | |||
| public: | |||
| // Default values, also used by python_bindings.cc | |||
| static const float kDefProbability; | |||
| explicit RandomHorizontalFlipWithBBoxOp(float probability = kDefProbability) : distribution_(probability) { | |||
| rnd_.seed(GetSeed()); | |||
| } | |||
| ~RandomHorizontalFlipWithBBoxOp() override = default; | |||
| // Provide stream operator for displaying it | |||
| friend std::ostream &operator<<(std::ostream &out, const RandomHorizontalFlipWithBBoxOp &so) { | |||
| so.Print(out); | |||
| return out; | |||
| } | |||
| void Print(std::ostream &out) const override { out << "RandomHorizontalFlipWithBBoxOp"; } | |||
| Status Compute(const TensorRow &input, TensorRow *output) override; | |||
| private: | |||
| std::mt19937 rnd_; | |||
| std::bernoulli_distribution distribution_; | |||
| }; | |||
| } // namespace dataset | |||
| } // namespace mindspore | |||
| #endif // DATASET_KERNELS_IMAGE_RANDOM_HORIZONTAL_FLIP_BBOX_OP_H_ | |||
| @@ -43,6 +43,36 @@ | |||
| } \ | |||
| } while (false) | |||
| #define BOUNDING_BOX_CHECK(input) \ | |||
| do { \ | |||
| uint32_t num_of_features = input[1]->shape()[1]; \ | |||
| if (num_of_features < 4) { \ | |||
| return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ | |||
| "Bounding boxes should be have at least 4 features"); \ | |||
| } \ | |||
| uint32_t num_of_boxes = input[1]->shape()[0]; \ | |||
| uint32_t img_h = input[0]->shape()[0]; \ | |||
| uint32_t img_w = input[0]->shape()[1]; \ | |||
| for (uint32_t i = 0; i < num_of_boxes; i++) { \ | |||
| uint32_t min_x = 0; \ | |||
| uint32_t min_y = 0; \ | |||
| uint32_t b_w = 0; \ | |||
| uint32_t b_h = 0; \ | |||
| input[1]->GetItemAt<uint32_t>(&min_x, {i, 0}); \ | |||
| input[1]->GetItemAt<uint32_t>(&min_y, {i, 1}); \ | |||
| input[1]->GetItemAt<uint32_t>(&b_w, {i, 2}); \ | |||
| input[1]->GetItemAt<uint32_t>(&b_h, {i, 3}); \ | |||
| if ((min_x + b_w > img_w) || (min_y + b_h > img_h)) { \ | |||
| return Status(StatusCode::kBoundingBoxOutOfBounds, __LINE__, __FILE__, \ | |||
| "At least one of the bounding boxes is out of bounds of the image."); \ | |||
| } \ | |||
| if (static_cast<int>(min_x) < 0 || static_cast<int>(min_y) < 0) { \ | |||
| return Status(StatusCode::kBoundingBoxOutOfBounds, __LINE__, __FILE__, \ | |||
| "At least one of the bounding boxes has negative min_x or min_y."); \ | |||
| } \ | |||
| } \ | |||
| } while (false) | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| // A class that does a computation on a Tensor | |||
| @@ -71,6 +71,8 @@ enum class StatusCode : char { | |||
| kTDTPushFailure = 8, | |||
| kFileNotExist = 9, | |||
| kProfilingError = 10, | |||
| kBoundingBoxOutOfBounds = 11, | |||
| kBoundingBoxInvalidShape = 12, | |||
| // Make this error code the last one. Add new error code above it. | |||
| kUnexpectedError = 127 | |||
| }; | |||
| @@ -45,7 +45,7 @@ import mindspore._c_dataengine as cde | |||
| from .utils import Inter, Border | |||
| from .validators import check_prob, check_crop, check_resize_interpolation, check_random_resize_crop, \ | |||
| check_normalize_c, check_random_crop, check_random_color_adjust, check_random_rotation, \ | |||
| check_resize, check_rescale, check_pad, check_cutout, check_uniform_augment_cpp | |||
| check_resize, check_rescale, check_pad, check_cutout, check_uniform_augment_cpp, check_bounding_box_augment_cpp | |||
| DE_C_INTER_MODE = {Inter.NEAREST: cde.InterpolationMode.DE_INTER_NEAREST_NEIGHBOUR, | |||
| Inter.LINEAR: cde.InterpolationMode.DE_INTER_LINEAR, | |||
| @@ -163,6 +163,21 @@ class RandomHorizontalFlip(cde.RandomHorizontalFlipOp): | |||
| super().__init__(prob) | |||
| class RandomHorizontalFlipWithBBox(cde.RandomHorizontalFlipWithBBoxOp): | |||
| """ | |||
| Flip the input image horizontally, randomly with a given probability. | |||
| Maintains data integrity by also flipping bounding boxes in an object detection pipeline. | |||
| Args: | |||
| prob (float): Probability of the image being flipped (default=0.5). | |||
| """ | |||
| @check_prob | |||
| def __init__(self, prob=0.5): | |||
| self.prob = prob | |||
| super().__init__(prob) | |||
| class RandomVerticalFlip(cde.RandomVerticalFlipOp): | |||
| """ | |||
| Flip the input image vertically, randomly with a given probability. | |||
| @@ -177,6 +192,21 @@ class RandomVerticalFlip(cde.RandomVerticalFlipOp): | |||
| super().__init__(prob) | |||
| class BoundingBoxAug(cde.BoundingBoxAugOp): | |||
| """ | |||
| Flip the input image vertically, randomly with a given probability. | |||
| Args: | |||
| transform: C++ operation (python OPs are not accepted). | |||
| ratio (float): Ratio of bounding boxes to apply augmentation on. Range: [0,1] (default=1). | |||
| """ | |||
| @check_bounding_box_augment_cpp | |||
| def __init__(self, transform, ratio=0.3): | |||
| self.ratio = ratio | |||
| self.transform = transform | |||
| super().__init__(transform, ratio) | |||
| class Resize(cde.ResizeOp): | |||
| """ | |||
| Resize the input image to the given size. | |||
| @@ -852,6 +852,32 @@ def check_uniform_augment_cpp(method): | |||
| return new_method | |||
| def check_bounding_box_augment_cpp(method): | |||
| """Wrapper method to check the parameters of BoundingBoxAugment cpp op.""" | |||
| @wraps(method) | |||
| def new_method(self, *args, **kwargs): | |||
| transform, ratio = (list(args) + 2 * [None])[:2] | |||
| if "transform" in kwargs: | |||
| transform = kwargs.get("transform") | |||
| if "ratio" in kwargs: | |||
| ratio = kwargs.get("ratio") | |||
| if ratio is not None: | |||
| check_value(ratio, [0., 1.]) | |||
| kwargs["ratio"] = ratio | |||
| else: | |||
| ratio = 0.3 | |||
| if not isinstance(ratio, float) and not isinstance(ratio, int): | |||
| raise ValueError("Ratio should be an int or float.") | |||
| if not isinstance(transform, TensorOp): | |||
| raise ValueError("Transform can only be a C++ operation.") | |||
| kwargs["transform"] = transform | |||
| kwargs["ratio"] = ratio | |||
| return method(self, **kwargs) | |||
| return new_method | |||
| def check_uniform_augment_py(method): | |||
| """Wrapper method to check the parameters of python UniformAugment op.""" | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>121.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>375</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>dog</name> | |||
| <pose>Frontal</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>55</xmin> | |||
| <ymin>34</ymin> | |||
| <xmax>624</xmax> | |||
| <ymax>555</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>123.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>375</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>car</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>1</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>42</xmin> | |||
| <ymin>6</ymin> | |||
| <xmax>610</xmax> | |||
| <ymax>600</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>129.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>375</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>dog</name> | |||
| <pose>Frontal</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>1328</xmin> | |||
| <ymin>431</ymin> | |||
| <xmax>2662</xmax> | |||
| <ymax>1695</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>32.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>281</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>train</name> | |||
| <pose>Frontal</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>1168</xmin> | |||
| <ymin>405</ymin> | |||
| <xmax>3270</xmax> | |||
| <ymax>2022</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>32.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>281</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>train</name> | |||
| <pose>Frontal</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>1168</xmin> | |||
| <ymin>405</ymin> | |||
| <xmax>3270</xmax> | |||
| <ymax>2022</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>33.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>366</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>person</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>1168</xmin> | |||
| <ymin>395</ymin> | |||
| <xmax>2859</xmax> | |||
| <ymax>2084</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>39.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>375</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>dog</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>684</xmin> | |||
| <ymin>311</ymin> | |||
| <xmax>3112</xmax> | |||
| <ymax>1820</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>42.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>335</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>person</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>1</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>874</xmin> | |||
| <ymin>152</ymin> | |||
| <xmax>2827</xmax> | |||
| <ymax>2000</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,39 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>61.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>333</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>train</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>25</xmin> | |||
| <ymin>40</ymin> | |||
| <xmax>641</xmax> | |||
| <ymax>613</ymax> | |||
| </bndbox> | |||
| </object> | |||
| <object> | |||
| <name>person</name> | |||
| <pose>Frontal</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>204</xmin> | |||
| <ymin>198</ymin> | |||
| <xmax>271</xmax> | |||
| <ymax>293</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,39 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>63.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>375</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>cat</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>0</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>23</xmin> | |||
| <ymin>17</ymin> | |||
| <xmax>565</xmax> | |||
| <ymax>591</ymax> | |||
| </bndbox> | |||
| </object> | |||
| <object> | |||
| <name>chair</name> | |||
| <pose>Frontal</pose> | |||
| <truncated>1</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>36</xmin> | |||
| <ymin>11</ymin> | |||
| <xmax>439</xmax> | |||
| <ymax>499</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1,27 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>68.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>375</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| <object> | |||
| <name>cat</name> | |||
| <pose>Unspecified</pose> | |||
| <truncated>1</truncated> | |||
| <difficult>0</difficult> | |||
| <bndbox> | |||
| <xmin>35</xmin> | |||
| <ymin>11</ymin> | |||
| <xmax>564</xmax> | |||
| <ymax>545</ymax> | |||
| </bndbox> | |||
| </object> | |||
| </annotation> | |||
| @@ -0,0 +1 @@ | |||
| invalidxml | |||
| @@ -0,0 +1,15 @@ | |||
| <annotation> | |||
| <folder>VOC2012</folder> | |||
| <filename>33.jpg</filename> | |||
| <source> | |||
| <database>simulate VOC2007 Database</database> | |||
| <annotation>simulate VOC2007</annotation> | |||
| <image>flickr</image> | |||
| </source> | |||
| <size> | |||
| <width>500</width> | |||
| <height>366</height> | |||
| <depth>3</depth> | |||
| </size> | |||
| <segmented>1</segmented> | |||
| </annotation> | |||
| @@ -0,0 +1 @@ | |||
| invalidxml | |||
| @@ -0,0 +1,11 @@ | |||
| 15 | |||
| 32 | |||
| 33 | |||
| 39 | |||
| 42 | |||
| 61 | |||
| 63 | |||
| 68 | |||
| 121 | |||
| 123 | |||
| 129 | |||
| @@ -0,0 +1 @@ | |||
| 15 | |||
| @@ -0,0 +1 @@ | |||
| 15 | |||
| @@ -0,0 +1 @@ | |||
| xmlnoobject | |||
| @@ -0,0 +1 @@ | |||
| 4176 | |||
| @@ -0,0 +1,10 @@ | |||
| 32 | |||
| 33 | |||
| 39 | |||
| 42 | |||
| 61 | |||
| 63 | |||
| 68 | |||
| 121 | |||
| 123 | |||
| 129 | |||
| @@ -0,0 +1 @@ | |||
| 15 | |||
| @@ -0,0 +1 @@ | |||
| 15 | |||
| @@ -0,0 +1,317 @@ | |||
| # Copyright 2020 Huawei Technologies Co., Ltd | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # ============================================================================== | |||
| """ | |||
| Testing the bounding box augment op in DE | |||
| """ | |||
| from enum import Enum | |||
| from mindspore import log as logger | |||
| import mindspore.dataset as ds | |||
| import mindspore.dataset.transforms.vision.c_transforms as c_vision | |||
| import matplotlib.pyplot as plt | |||
| import matplotlib.patches as patches | |||
| import numpy as np | |||
| GENERATE_GOLDEN = False | |||
| DATA_DIR = "../data/dataset/testVOC2012_2" | |||
| class BoxType(Enum): | |||
| """ | |||
| Defines box types for test cases | |||
| """ | |||
| WidthOverflow = 1 | |||
| HeightOverflow = 2 | |||
| NegativeXY = 3 | |||
| OnEdge = 4 | |||
| WrongShape = 5 | |||
| class AddBadAnnotation: # pylint: disable=too-few-public-methods | |||
| """ | |||
| Used to add erroneous bounding boxes to object detection pipelines. | |||
| Usage: | |||
| >>> # Adds a box that covers the whole image. Good for testing edge cases | |||
| >>> de = de.map(input_columns=["image", "annotation"], | |||
| >>> output_columns=["image", "annotation"], | |||
| >>> operations=AddBadAnnotation(BoxType.OnEdge)) | |||
| """ | |||
| def __init__(self, box_type): | |||
| self.box_type = box_type | |||
| def __call__(self, img, bboxes): | |||
| """ | |||
| Used to generate erroneous bounding box examples on given img. | |||
| :param img: image where the bounding boxes are. | |||
| :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format | |||
| :return: bboxes with bad examples added | |||
| """ | |||
| height = img.shape[0] | |||
| width = img.shape[1] | |||
| if self.box_type == BoxType.WidthOverflow: | |||
| # use box that overflows on width | |||
| return img, np.array([[0, 0, width + 1, height, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.HeightOverflow: | |||
| # use box that overflows on height | |||
| return img, np.array([[0, 0, width, height + 1, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.NegativeXY: | |||
| # use box with negative xy | |||
| return img, np.array([[-10, -10, width, height, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.OnEdge: | |||
| # use box that covers the whole image | |||
| return img, np.array([[0, 0, width, height, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.WrongShape: | |||
| # use box that covers the whole image | |||
| return img, np.array([[0, 0, width - 1]]).astype(np.uint32) | |||
| return img, bboxes | |||
| def h_flip(image): | |||
| """ | |||
| Apply the random_horizontal | |||
| """ | |||
| # with the seed provided in this test case, it will always flip. | |||
| # that's why we flip here too | |||
| image = image[:, ::-1, :] | |||
| return image | |||
| def check_bad_box(data, box_type, expected_error): | |||
| """ | |||
| :param data: de object detection pipeline | |||
| :param box_type: type of bad box | |||
| :param expected_error: error expected to get due to bad box | |||
| :return: None | |||
| """ | |||
| try: | |||
| test_op = c_vision.BoundingBoxAug(c_vision.RandomHorizontalFlip(1), | |||
| 1) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) | |||
| data = data.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to use width overflow | |||
| data = data.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=AddBadAnnotation(box_type)) # Add column for "annotation" | |||
| # map to apply ops | |||
| data = data.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| for _, _ in enumerate(data.create_dict_iterator()): | |||
| break | |||
| except RuntimeError as error: | |||
| logger.info("Got an exception in DE: {}".format(str(error))) | |||
| assert expected_error in str(error) | |||
| def fix_annotate(bboxes): | |||
| """ | |||
| Fix annotations to format followed by mindspore. | |||
| :param bboxes: in [label, x_min, y_min, w, h, truncate, difficult] format | |||
| :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format | |||
| """ | |||
| for bbox in bboxes: | |||
| tmp = bbox[0] | |||
| bbox[0] = bbox[1] | |||
| bbox[1] = bbox[2] | |||
| bbox[2] = bbox[3] | |||
| bbox[3] = bbox[4] | |||
| bbox[4] = tmp | |||
| return bboxes | |||
| def add_bounding_boxes(axis, bboxes): | |||
| """ | |||
| :param axis: axis to modify | |||
| :param bboxes: bounding boxes to draw on the axis | |||
| :return: None | |||
| """ | |||
| for bbox in bboxes: | |||
| rect = patches.Rectangle((bbox[0], bbox[1]), | |||
| bbox[2], bbox[3], | |||
| linewidth=1, edgecolor='r', facecolor='none') | |||
| # Add the patch to the Axes | |||
| axis.add_patch(rect) | |||
| def visualize(unaugmented_data, augment_data): | |||
| """ | |||
| :param unaugmented_data: original data | |||
| :param augment_data: data after augmentations | |||
| :return: None | |||
| """ | |||
| for idx, (un_aug_item, aug_item) in \ | |||
| enumerate(zip(unaugmented_data.create_dict_iterator(), | |||
| augment_data.create_dict_iterator())): | |||
| axis = plt.subplot(141) | |||
| plt.imshow(un_aug_item["image"]) | |||
| add_bounding_boxes(axis, un_aug_item["annotation"]) # add Orig BBoxes | |||
| plt.title("Original" + str(idx + 1)) | |||
| logger.info("Original ", str(idx + 1), " :", un_aug_item["annotation"]) | |||
| axis = plt.subplot(142) | |||
| plt.imshow(aug_item["image"]) | |||
| add_bounding_boxes(axis, aug_item["annotation"]) # add AugBBoxes | |||
| plt.title("Augmented" + str(idx + 1)) | |||
| logger.info("Augmented ", str(idx + 1), " ", aug_item["annotation"], "\n") | |||
| plt.show() | |||
| def test_bounding_box_augment_with_rotation_op(plot=False): | |||
| """ | |||
| Test BoundingBoxAugment op | |||
| Prints images side by side with and without Aug applied + bboxes to compare and test | |||
| """ | |||
| logger.info("test_bounding_box_augment_with_rotation_op") | |||
| data_voc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| test_op = c_vision.BoundingBoxAug(c_vision.RandomRotation(90), 1) | |||
| # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) | |||
| # maps to fix annotations to minddata standard | |||
| data_voc1 = data_voc1.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| if plot: | |||
| visualize(data_voc1, data_voc2) | |||
| def test_bounding_box_augment_with_crop_op(plot=False): | |||
| """ | |||
| Test BoundingBoxAugment op | |||
| Prints images side by side with and without Aug applied + bboxes to compare and test | |||
| """ | |||
| logger.info("test_bounding_box_augment_with_crop_op") | |||
| data_voc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| test_op = c_vision.BoundingBoxAug(c_vision.RandomCrop(90), 1) | |||
| # maps to fix annotations to minddata standard | |||
| data_voc1 = data_voc1.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| if plot: | |||
| visualize(data_voc1, data_voc2) | |||
| def test_bounding_box_augment_valid_ratio_c(plot=False): | |||
| """ | |||
| Test RandomHorizontalFlipWithBBox op | |||
| Prints images side by side with and without Aug applied + bboxes to compare and test | |||
| """ | |||
| logger.info("test_bounding_box_augment_valid_ratio_c") | |||
| data_voc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| test_op = c_vision.BoundingBoxAug(c_vision.RandomHorizontalFlip(1), 0.9) | |||
| # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) | |||
| # maps to fix annotations to minddata standard | |||
| data_voc1 = data_voc1.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| if plot: | |||
| visualize(data_voc1, data_voc2) | |||
| def test_bounding_box_augment_invalid_ratio_c(): | |||
| """ | |||
| Test RandomHorizontalFlipWithBBox op with invalid input probability | |||
| """ | |||
| logger.info("test_bounding_box_augment_invalid_ratio_c") | |||
| data_voc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| try: | |||
| # ratio range is from 0 - 1 | |||
| test_op = c_vision.BoundingBoxAug(c_vision.RandomHorizontalFlip(1), 1.5) | |||
| # maps to fix annotations to minddata standard | |||
| data_voc1 = data_voc1.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| except ValueError as error: | |||
| logger.info("Got an exception in DE: {}".format(str(error))) | |||
| assert "Input is not" in str(error) | |||
| def test_bounding_box_augment_invalid_bounds_c(): | |||
| """ | |||
| Test BoundingBoxAugment op with invalid bboxes. | |||
| """ | |||
| logger.info("test_bounding_box_augment_invalid_bounds_c") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.WidthOverflow, "bounding boxes is out of bounds of the image") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.HeightOverflow, "bounding boxes is out of bounds of the image") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.NegativeXY, "min_x") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.WrongShape, "4 features") | |||
| if __name__ == "__main__": | |||
| # set to false to not show plots | |||
| test_bounding_box_augment_with_rotation_op(False) | |||
| test_bounding_box_augment_with_crop_op(False) | |||
| test_bounding_box_augment_valid_ratio_c(False) | |||
| test_bounding_box_augment_invalid_ratio_c() | |||
| test_bounding_box_augment_invalid_bounds_c() | |||
| @@ -0,0 +1,281 @@ | |||
| # Copyright 2020 Huawei Technologies Co., Ltd | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # ============================================================================== | |||
| """ | |||
| Testing the random horizontal flip with bounding boxes op in DE | |||
| """ | |||
| from enum import Enum | |||
| from mindspore import log as logger | |||
| import mindspore.dataset as ds | |||
| import mindspore.dataset.transforms.vision.c_transforms as c_vision | |||
| import matplotlib.pyplot as plt | |||
| import matplotlib.patches as patches | |||
| import numpy as np | |||
| GENERATE_GOLDEN = False | |||
| DATA_DIR = "../data/dataset/testVOC2012_2" | |||
| class BoxType(Enum): | |||
| """ | |||
| Defines box types for test cases | |||
| """ | |||
| WidthOverflow = 1 | |||
| HeightOverflow = 2 | |||
| NegativeXY = 3 | |||
| OnEdge = 4 | |||
| WrongShape = 5 | |||
| class AddBadAnnotation: # pylint: disable=too-few-public-methods | |||
| """ | |||
| Used to add erroneous bounding boxes to object detection pipelines. | |||
| Usage: | |||
| >>> # Adds a box that covers the whole image. Good for testing edge cases | |||
| >>> de = de.map(input_columns=["image", "annotation"], | |||
| >>> output_columns=["image", "annotation"], | |||
| >>> operations=AddBadAnnotation(BoxType.OnEdge)) | |||
| """ | |||
| def __init__(self, box_type): | |||
| self.box_type = box_type | |||
| def __call__(self, img, bboxes): | |||
| """ | |||
| Used to generate erroneous bounding box examples on given img. | |||
| :param img: image where the bounding boxes are. | |||
| :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format | |||
| :return: bboxes with bad examples added | |||
| """ | |||
| height = img.shape[0] | |||
| width = img.shape[1] | |||
| if self.box_type == BoxType.WidthOverflow: | |||
| # use box that overflows on width | |||
| return img, np.array([[0, 0, width + 1, height, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.HeightOverflow: | |||
| # use box that overflows on height | |||
| return img, np.array([[0, 0, width, height + 1, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.NegativeXY: | |||
| # use box with negative xy | |||
| return img, np.array([[-10, -10, width, height, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.OnEdge: | |||
| # use box that covers the whole image | |||
| return img, np.array([[0, 0, width, height, 0, 0, 0]]).astype(np.uint32) | |||
| if self.box_type == BoxType.WrongShape: | |||
| # use box that covers the whole image | |||
| return img, np.array([[0, 0, width - 1]]).astype(np.uint32) | |||
| return img, bboxes | |||
| def h_flip(image): | |||
| """ | |||
| Apply the random_horizontal | |||
| """ | |||
| # with the seed provided in this test case, it will always flip. | |||
| # that's why we flip here too | |||
| image = image[:, ::-1, :] | |||
| return image | |||
| def check_bad_box(data, box_type, expected_error): | |||
| """ | |||
| :param data: de object detection pipeline | |||
| :param box_type: type of bad box | |||
| :param expected_error: error expected to get due to bad box | |||
| :return: None | |||
| """ | |||
| # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) | |||
| try: | |||
| test_op = c_vision.RandomHorizontalFlipWithBBox(1) | |||
| data = data.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to use width overflow | |||
| data = data.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=AddBadAnnotation(box_type)) # Add column for "annotation" | |||
| # map to apply ops | |||
| data = data.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| for _, _ in enumerate(data.create_dict_iterator()): | |||
| break | |||
| except RuntimeError as error: | |||
| logger.info("Got an exception in DE: {}".format(str(error))) | |||
| assert expected_error in str(error) | |||
| def fix_annotate(bboxes): | |||
| """ | |||
| Fix annotations to format followed by mindspore. | |||
| :param bboxes: in [label, x_min, y_min, w, h, truncate, difficult] format | |||
| :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format | |||
| """ | |||
| for bbox in bboxes: | |||
| tmp = bbox[0] | |||
| bbox[0] = bbox[1] | |||
| bbox[1] = bbox[2] | |||
| bbox[2] = bbox[3] | |||
| bbox[3] = bbox[4] | |||
| bbox[4] = tmp | |||
| return bboxes | |||
| def add_bounding_boxes(axis, bboxes): | |||
| """ | |||
| :param axis: axis to modify | |||
| :param bboxes: bounding boxes to draw on the axis | |||
| :return: None | |||
| """ | |||
| for bbox in bboxes: | |||
| rect = patches.Rectangle((bbox[0], bbox[1]), | |||
| bbox[2], bbox[3], | |||
| linewidth=1, edgecolor='r', facecolor='none') | |||
| # Add the patch to the Axes | |||
| axis.add_patch(rect) | |||
| def visualize(unaugmented_data, augment_data): | |||
| """ | |||
| :param unaugmented_data: original data | |||
| :param augment_data: data after augmentations | |||
| :return: None | |||
| """ | |||
| for idx, (un_aug_item, aug_item) in \ | |||
| enumerate(zip(unaugmented_data.create_dict_iterator(), | |||
| augment_data.create_dict_iterator())): | |||
| axis = plt.subplot(141) | |||
| plt.imshow(un_aug_item["image"]) | |||
| add_bounding_boxes(axis, un_aug_item["annotation"]) # add Orig BBoxes | |||
| plt.title("Original" + str(idx + 1)) | |||
| logger.info("Original ", str(idx + 1), " :", un_aug_item["annotation"]) | |||
| axis = plt.subplot(142) | |||
| plt.imshow(aug_item["image"]) | |||
| add_bounding_boxes(axis, aug_item["annotation"]) # add AugBBoxes | |||
| plt.title("Augmented" + str(idx + 1)) | |||
| logger.info("Augmented ", str(idx + 1), " ", aug_item["annotation"], "\n") | |||
| plt.show() | |||
| def test_random_horizontal_bbox_op(plot=False): | |||
| """ | |||
| Test RandomHorizontalFlipWithBBox op | |||
| Prints images side by side with and without Aug applied + bboxes to compare and test | |||
| """ | |||
| logger.info("test_random_horizontal_bbox_c") | |||
| data_voc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) | |||
| test_op = c_vision.RandomHorizontalFlipWithBBox(1) | |||
| # maps to fix annotations to minddata standard | |||
| data_voc1 = data_voc1.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| if plot: | |||
| visualize(data_voc1, data_voc2) | |||
| def test_random_horizontal_bbox_valid_prob_c(plot=False): | |||
| """ | |||
| Test RandomHorizontalFlipWithBBox op | |||
| Prints images side by side with and without Aug applied + bboxes to compare and test | |||
| """ | |||
| logger.info("test_random_horizontal_bbox_valid_prob_c") | |||
| data_voc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) | |||
| test_op = c_vision.RandomHorizontalFlipWithBBox(0.3) | |||
| # maps to fix annotations to minddata standard | |||
| data_voc1 = data_voc1.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| if plot: | |||
| visualize(data_voc1, data_voc2) | |||
| def test_random_horizontal_bbox_invalid_prob_c(): | |||
| """ | |||
| Test RandomHorizontalFlipWithBBox op with invalid input probability | |||
| """ | |||
| logger.info("test_random_horizontal_bbox_invalid_prob_c") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| try: | |||
| # Note: Valid range of prob should be [0.0, 1.0] | |||
| test_op = c_vision.RandomHorizontalFlipWithBBox(1.5) | |||
| data_voc2 = data_voc2.map(input_columns=["annotation"], | |||
| output_columns=["annotation"], | |||
| operations=fix_annotate) | |||
| # map to apply ops | |||
| data_voc2 = data_voc2.map(input_columns=["image", "annotation"], | |||
| output_columns=["image", "annotation"], | |||
| columns_order=["image", "annotation"], | |||
| operations=[test_op]) # Add column for "annotation" | |||
| except ValueError as error: | |||
| logger.info("Got an exception in DE: {}".format(str(error))) | |||
| assert "Input is not" in str(error) | |||
| def test_random_horizontal_bbox_invalid_bounds_c(): | |||
| """ | |||
| Test RandomHorizontalFlipWithBBox op with invalid bounding boxes | |||
| """ | |||
| logger.info("test_random_horizontal_bbox_invalid_bounds_c") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.WidthOverflow, "bounding boxes is out of bounds of the image") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.HeightOverflow, "bounding boxes is out of bounds of the image") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.NegativeXY, "min_x") | |||
| data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) | |||
| check_bad_box(data_voc2, BoxType.WrongShape, "4 features") | |||
| if __name__ == "__main__": | |||
| # set to false to not show plots | |||
| test_random_horizontal_bbox_op(False) | |||
| test_random_horizontal_bbox_valid_prob_c(False) | |||
| test_random_horizontal_bbox_invalid_prob_c() | |||
| test_random_horizontal_bbox_invalid_bounds_c() | |||