/** * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_ #define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_ #include #include #include #include #include #include "include/api/status.h" #include "minddata/dataset/include/constants.h" #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/vision_lite.h" namespace mindspore { namespace dataset { // Transform operations for performing computer vision. namespace vision { // Transform Op classes (in alphabetical order) class AutoContrastOperation; class BoundingBoxAugmentOperation; class CutMixBatchOperation; class CutOutOperation; class EqualizeOperation; class HwcToChwOperation; class InvertOperation; class MixUpBatchOperation; class NormalizePadOperation; class PadOperation; class RandomAffineOperation; class RandomColorOperation; class RandomColorAdjustOperation; class RandomCropOperation; class RandomCropDecodeResizeOperation; class RandomCropWithBBoxOperation; class RandomHorizontalFlipOperation; class RandomHorizontalFlipWithBBoxOperation; class RandomPosterizeOperation; class RandomResizeOperation; class RandomResizeWithBBoxOperation; class RandomResizedCropOperation; class RandomResizedCropWithBBoxOperation; class RandomRotationOperation; class RandomSelectSubpolicyOperation; class RandomSharpnessOperation; class RandomSolarizeOperation; class RandomVerticalFlipOperation; class RandomVerticalFlipWithBBoxOperation; class RescaleOperation; class ResizeWithBBoxOperation; class RgbaToBgrOperation; class RgbaToRgbOperation; class SoftDvppDecodeRandomCropResizeJpegOperation; class SoftDvppDecodeResizeJpegOperation; class SwapRedBlueOperation; class UniformAugOperation; /// \brief Function to create a AutoContrast TensorOperation. /// \notes Apply automatic contrast on input image. /// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 100. /// \param[in] ignore Pixel values to ignore. /// \return Shared pointer to the current TensorOperation. std::shared_ptr AutoContrast(float cutoff = 0.0, std::vector ignore = {}); /// \brief Function to create a BoundingBoxAugment TensorOperation. /// \notes Apply a given image transform on a random selection of bounding box regions of a given image. /// \param[in] transform A TensorOperation transform. /// \param[in] ratio Ratio of bounding boxes to apply augmentation on. Range: [0, 1] (default=0.3). /// \return Shared pointer to the current TensorOperation. std::shared_ptr BoundingBoxAugment(std::shared_ptr transform, float ratio = 0.3); /// \brief Function to apply CutMix on a batch of images /// \notes Masks a random section of each image with the corresponding part of another randomly /// selected image in that batch /// \param[in] image_batch_format The format of the batch /// \param[in] alpha The hyperparameter of beta distribution (default = 1.0) /// \param[in] prob The probability by which CutMix is applied to each image (default = 1.0) /// \return Shared pointer to the current TensorOp std::shared_ptr CutMixBatch(ImageBatchFormat image_batch_format, float alpha = 1.0, float prob = 1.0); /// \brief Function to create a CutOut TensorOp /// \notes Randomly cut (mask) out a given number of square patches from the input image /// \param[in] length Integer representing the side length of each square patch /// \param[in] num_patches Integer representing the number of patches to be cut out of an image /// \return Shared pointer to the current TensorOp std::shared_ptr CutOut(int32_t length, int32_t num_patches = 1); /// \brief Function to create a Equalize TensorOperation. /// \notes Apply histogram equalization on input image. /// \return Shared pointer to the current TensorOperation. std::shared_ptr Equalize(); /// \brief Function to create a HwcToChw TensorOperation. /// \notes Transpose the input image; shape (H, W, C) to shape (C, H, W). /// \return Shared pointer to the current TensorOperation. std::shared_ptr HWC2CHW(); /// \brief Function to create a Invert TensorOperation. /// \notes Apply invert on input image in RGB mode. /// \return Shared pointer to the current TensorOperation. std::shared_ptr Invert(); /// \brief Function to create a MixUpBatch TensorOperation. /// \notes Apply MixUp transformation on an input batch of images and labels. The labels must be in /// one-hot format and Batch must be called before calling this function. /// \param[in] alpha hyperparameter of beta distribution (default = 1.0) /// \return Shared pointer to the current TensorOperation. std::shared_ptr MixUpBatch(float alpha = 1); /// \brief Function to create a NormalizePad TensorOperation. /// \notes Normalize the input image with respect to mean and standard deviation and pad an extra /// channel with value zero. /// \param[in] mean A vector of mean values for each channel, w.r.t channel order. /// The mean values must be in range [0.0, 255.0]. /// \param[in] std A vector of standard deviations for each channel, w.r.t. channel order. /// The standard deviation values must be in range (0.0, 255.0] /// \param[in] dtype The output datatype of Tensor. /// The standard deviation values must be "float32" or "float16"(default = "float32") /// \return Shared pointer to the current TensorOperation. std::shared_ptr NormalizePad(const std::vector &mean, const std::vector &std, const std::string &dtype = "float32"); /// \brief Function to create a Pad TensorOp /// \notes Pads the image according to padding parameters /// \param[in] padding A vector representing the number of pixels to pad the image /// If vector has one value, it pads all sides of the image with that value. /// If vector has two values, it pads left and top with the first and /// right and bottom with the second value. /// If vector has four values, it pads left, top, right, and bottom with /// those values respectively. /// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is /// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. If 3 values are provided, /// it is used to fill R, G, B channels respectively. /// \param[in] padding_mode The method of padding (default=BorderType.kConstant) /// Can be any of /// [BorderType.kConstant, BorderType.kEdge, BorderType.kReflect, BorderType.kSymmetric] /// - BorderType.kConstant, means it fills the border with constant values /// - BorderType.kEdge, means it pads with the last value on the edge /// - BorderType.kReflect, means it reflects the values on the edge omitting the last value of edge /// - BorderType.kSymmetric, means it reflects the values on the edge repeating the last value of edge /// \return Shared pointer to the current TensorOp std::shared_ptr Pad(std::vector padding, std::vector fill_value = {0}, BorderType padding_mode = BorderType::kConstant); /// \brief Function to create a RandomAffine TensorOperation. /// \notes Applies a Random Affine transformation on input image in RGB or Greyscale mode. /// \param[in] degrees A float vector of size 2, representing the starting and ending degree /// \param[in] translate_range A float vector of size 2 or 4, representing percentages of translation on x and y axes. /// if size is 2, (min_dx, max_dx, 0, 0) /// if size is 4, (min_dx, max_dx, min_dy, max_dy) /// all values are in range [-1, 1] /// \param[in] scale_range A float vector of size 2, representing the starting and ending scales in the range. /// \param[in] shear_ranges A float vector of size 2 or 4, representing the starting and ending shear degrees /// vertically and horizontally. /// if size is 2, (min_shear_x, max_shear_x, 0, 0) /// if size is 4, (min_shear_x, max_shear_x, min_shear_y, max_shear_y) /// \param[in] interpolation An enum for the mode of interpolation /// \param[in] fill_value A vector representing the value to fill the area outside the transform /// in the output image. If 1 value is provided, it is used for all RGB channels. /// If 3 values are provided, it is used to fill R, G, B channels respectively. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomAffine( const std::vector °rees, const std::vector &translate_range = {0.0, 0.0, 0.0, 0.0}, const std::vector &scale_range = {1.0, 1.0}, const std::vector &shear_ranges = {0.0, 0.0, 0.0, 0.0}, InterpolationMode interpolation = InterpolationMode::kNearestNeighbour, const std::vector &fill_value = {0, 0, 0}); /// \brief Blends an image with its grayscale version with random weights /// t and 1 - t generated from a given range. If the range is trivial /// then the weights are determinate and t equals the bound of the interval /// \param[in] t_lb Lower bound on the range of random weights /// \param[in] t_lb Upper bound on the range of random weights /// \return Shared pointer to the current TensorOp std::shared_ptr RandomColor(float t_lb, float t_ub); /// \brief Randomly adjust the brightness, contrast, saturation, and hue of the input image /// \param[in] brightness Brightness adjustment factor. Must be a vector of one or two values /// if it's a vector of two values it needs to be in the form of [min, max]. Default value is {1, 1} /// \param[in] contrast Contrast adjustment factor. Must be a vector of one or two values /// if it's a vector of two values it needs to be in the form of [min, max]. Default value is {1, 1} /// \param[in] saturation Saturation adjustment factor. Must be a vector of one or two values /// if it's a vector of two values it needs to be in the form of [min, max]. Default value is {1, 1} /// \param[in] hue Brightness adjustment factor. Must be a vector of one or two values /// if it's a vector of two values it must be in the form of [min, max] where -0.5 <= min <= max <= 0.5 /// Default value is {0, 0} /// \return Shared pointer to the current TensorOp std::shared_ptr RandomColorAdjust(std::vector brightness = {1.0, 1.0}, std::vector contrast = {1.0, 1.0}, std::vector saturation = {1.0, 1.0}, std::vector hue = {0.0, 0.0}); /// \brief Function to create a RandomCrop TensorOperation. /// \notes Crop the input image at a random location. /// \param[in] size A vector representing the output size of the cropped image. /// If size is a single value, a square crop of size (size, size) is returned. /// If size has 2 values, it should be (height, width). /// \param[in] padding A vector representing the number of pixels to pad the image /// If vector has one value, it pads all sides of the image with that value. /// If vector has two values, it pads left and top with the first and /// right and bottom with the second value. /// If vector has four values, it pads left, top, right, and bottom with /// those values respectively. /// \param[in] pad_if_needed A boolean whether to pad the image if either side is smaller than /// the given output size. /// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is /// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. /// If 3 values are provided, it is used to fill R, G, B channels respectively. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomCrop(std::vector size, std::vector padding = {0, 0, 0, 0}, bool pad_if_needed = false, std::vector fill_value = {0, 0, 0}, BorderType padding_mode = BorderType::kConstant); /// \brief Function to create a RandomCropDecodeResize TensorOperation. /// \notes Equivalent to RandomResizedCrop, but crops before decodes. /// \param[in] size A vector representing the output size of the cropped image. /// If size is a single value, a square crop of size (size, size) is returned. /// If size has 2 values, it should be (height, width). /// \param[in] scale Range [min, max) of respective size of the /// original size to be cropped (default=(0.08, 1.0)) /// \param[in] ratio Range [min, max) of aspect ratio to be /// cropped (default=(3. / 4., 4. / 3.)) /// \param[in] interpolation An enum for the mode of interpolation /// \param[in] The maximum number of attempts to propose a valid crop_area (default=10). /// If exceeded, fall back to use center_crop instead. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomCropDecodeResize( std::vector size, std::vector scale = {0.08, 1.0}, std::vector ratio = {3. / 4, 4. / 3}, InterpolationMode interpolation = InterpolationMode::kLinear, int32_t max_attempts = 10); /// \brief Function to create a RandomCropWithBBox TensorOperation. /// \Crop the input image at a random location and adjust bounding boxes accordingly. /// \param[in] size A vector representing the output size of the cropped image. /// If size is a single value, a square crop of size (size, size) is returned. /// If size has 2 values, it should be (height, width). /// \param[in] padding A vector representing the number of pixels to pad the image /// If vector has one value, it pads all sides of the image with that value. /// If vector has two values, it pads left and top with the first and /// right and bottom with the second value. /// If vector has four values, it pads left, top, right, and bottom with /// those values respectively. /// \param[in] pad_if_needed A boolean whether to pad the image if either side is smaller than /// the given output size. /// \param[in] fill_value A vector representing the pixel intensity of the borders if the padding_mode is /// BorderType.kConstant. If 1 value is provided, it is used for all RGB channels. /// If 3 values are provided, it is used to fill R, G, B channels respectively. /// \param[in] padding_mode The method of padding (default=BorderType::kConstant).It can be any of /// [BorderType::kConstant, BorderType::kEdge, BorderType::kReflect, BorderType::kSymmetric]. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomCropWithBBox(std::vector size, std::vector padding = {0, 0, 0, 0}, bool pad_if_needed = false, std::vector fill_value = {0, 0, 0}, BorderType padding_mode = BorderType::kConstant); /// \brief Function to create a RandomHorizontalFlip TensorOperation. /// \notes Tensor operation to perform random horizontal flip. /// \param[in] prob A float representing the probability of flip. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomHorizontalFlip(float prob = 0.5); /// \brief Function to create a RandomHorizontalFlipWithBBox TensorOperation. /// \notes Flip the input image horizontally, randomly with a given probability and adjust bounding boxes accordingly. /// \param[in] prob A float representing the probability of flip. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomHorizontalFlipWithBBox(float prob = 0.5); /// \brief Function to create a RandomPosterize TensorOperation. /// \notes Tensor operation to perform random posterize. /// \param[in] bit_range - uint8_t vector representing the minimum and maximum bit in range. (Default={4, 8}) /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomPosterize(const std::vector &bit_range = {4, 8}); /// \brief Function to create a RandomResize TensorOperation. /// \notes Resize the input image using a randomly selected interpolation mode. /// \param[in] size A vector representing the output size of the resized image. /// If size is a single value, the smaller edge of the image will be resized to this value with // the same image aspect ratio. If size has 2 values, it should be (height, width). std::shared_ptr RandomResize(std::vector size); /// \brief Function to create a RandomResizeWithBBox TensorOperation. /// \notes Resize the input image using a randomly selected interpolation mode and adjust /// bounding boxes accordingly. /// \param[in] size A vector representing the output size of the resized image. /// If size is a single value, the smaller edge of the image will be resized to this value with // the same image aspect ratio. If size has 2 values, it should be (height, width). std::shared_ptr RandomResizeWithBBox(std::vector size); /// \brief Function to create a RandomResizedCrop TensorOperation. /// \notes Crop the input image to a random size and aspect ratio. /// \param[in] size A vector representing the output size of the cropped image. /// If size is a single value, a square crop of size (size, size) is returned. /// If size has 2 values, it should be (height, width). /// \param[in] scale Range [min, max) of respective size of the original /// size to be cropped (default=(0.08, 1.0)) /// \param[in] ratio Range [min, max) of aspect ratio to be cropped /// (default=(3. / 4., 4. / 3.)). /// \param[in] interpolation Image interpolation mode (default=InterpolationMode::kLinear) /// \param[in] max_attempts The maximum number of attempts to propose a valid /// crop_area (default=10). If exceeded, fall back to use center_crop instead. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomResizedCrop( std::vector size, std::vector scale = {0.08, 1.0}, std::vector ratio = {3. / 4., 4. / 3.}, InterpolationMode interpolation = InterpolationMode::kLinear, int32_t max_attempts = 10); /// \brief Function to create a RandomResizedCropWithBBox TensorOperation. /// \notes Crop the input image to a random size and aspect ratio. /// \param[in] size A vector representing the output size of the cropped image. /// If size is a single value, a square crop of size (size, size) is returned. /// If size has 2 values, it should be (height, width). /// \param[in] scale Range [min, max) of respective size of the original /// size to be cropped (default=(0.08, 1.0)) /// \param[in] ratio Range [min, max) of aspect ratio to be cropped /// (default=(3. / 4., 4. / 3.)). /// \param[in] interpolation Image interpolation mode (default=InterpolationMode::kLinear) /// \param[in] max_attempts The maximum number of attempts to propose a valid /// crop_area (default=10). If exceeded, fall back to use center_crop instead. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomResizedCropWithBBox( std::vector size, std::vector scale = {0.08, 1.0}, std::vector ratio = {3. / 4., 4. / 3.}, InterpolationMode interpolation = InterpolationMode::kLinear, int32_t max_attempts = 10); /// \brief Function to create a RandomRotation TensorOp /// \notes Rotates the image according to parameters /// \param[in] degrees A float vector of size, representing the starting and ending degree /// \param[in] resample An enum for the mode of interpolation /// \param[in] expand A boolean representing whether the image is expanded after rotation /// \param[in] center A float vector of size 2, representing the x and y center of rotation. /// \param[in] fill_value A vector representing the value to fill the area outside the transform /// in the output image. If 1 value is provided, it is used for all RGB channels. /// If 3 values are provided, it is used to fill R, G, B channels respectively. /// \return Shared pointer to the current TensorOp std::shared_ptr RandomRotation( std::vector degrees, InterpolationMode resample = InterpolationMode::kNearestNeighbour, bool expand = false, std::vector center = {-1, -1}, std::vector fill_value = {0, 0, 0}); /// \brief Function to create a RandomSelectSubpolicy TensorOperation. /// \notes Choose a random sub-policy from a list to be applied on the input image. A sub-policy is a list of tuples /// (op, prob), where op is a TensorOp operation and prob is the probability that this op will be applied. Once /// a sub-policy is selected, each op within the subpolicy with be applied in sequence according to its probability. /// \param[in] policy Vector of sub-policies to choose from. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomSelectSubpolicy( std::vector, double>>> policy); /// \brief Function to create a RandomSharpness TensorOperation. /// \notes Tensor operation to perform random sharpness. /// \param[in] degrees A float vector of size 2, representing the starting and ending degree to uniformly /// sample from, to select a degree to adjust sharpness. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomSharpness(std::vector degrees = {0.1, 1.9}); /// \brief Function to create a RandomSolarize TensorOperation. /// \notes Invert pixels randomly within specified range. If min=max, it is a single fixed magnitude operation /// to inverts all pixel above that threshold /// \param[in] threshold A vector with two elements specifying the pixel range to invert. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomSolarize(std::vector threshold = {0, 255}); /// \brief Function to create a RandomVerticalFlip TensorOperation. /// \notes Tensor operation to perform random vertical flip. /// \param[in] prob A float representing the probability of flip. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomVerticalFlip(float prob = 0.5); /// \brief Function to create a RandomVerticalFlipWithBBox TensorOperation. /// \notes Flip the input image vertically, randomly with a given probability and adjust bounding boxes accordingly. /// \param[in] prob A float representing the probability of flip. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RandomVerticalFlipWithBBox(float prob = 0.5); /// \brief Function to create a RescaleOperation TensorOperation. /// \notes Tensor operation to rescale the input image. /// \param[in] rescale Rescale factor. /// \param[in] shift Shift factor. /// \return Shared pointer to the current TensorOperation. std::shared_ptr Rescale(float rescale, float shift); /// \brief Function to create a ResizeWithBBox TensorOperation. /// \notes Resize the input image to the given size and adjust bounding boxes accordingly. /// \param[in] size The output size of the resized image. /// If size is an integer, smaller edge of the image will be resized to this value with the same image aspect ratio. /// If size is a sequence of length 2, it should be (height, width). /// \param[in] interpolation An enum for the mode of interpolation (default=InterpolationMode::kLinear). /// \return Shared pointer to the current TensorOperation. std::shared_ptr ResizeWithBBox(std::vector size, InterpolationMode interpolation = InterpolationMode::kLinear); /// \brief Function to create a RgbaToBgr TensorOperation. /// \notes Changes the input 4 channel RGBA tensor to 3 channel BGR. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RGBA2BGR(); /// \brief Function to create a RgbaToRgb TensorOperation. /// \notes Changes the input 4 channel RGBA tensor to 3 channel RGB. /// \return Shared pointer to the current TensorOperation. std::shared_ptr RGBA2RGB(); /// \brief Function to create a SoftDvppDecodeRandomCropResizeJpeg TensorOperation. /// \notes Tensor operation to decode, random crop and resize JPEG image using the simulation algorithm of /// Ascend series chip DVPP module. The usage scenario is consistent with SoftDvppDecodeResizeJpeg. /// The input image size should be in range [32*32, 8192*8192]. /// The zoom-out and zoom-in multiples of the image length and width should in the range [1/32, 16]. /// Only images with an even resolution can be output. The output of odd resolution is not supported. /// \param[in] size A vector representing the output size of the resized image. /// If size is a single value, smaller edge of the image will be resized to this value with /// the same image aspect ratio. If size has 2 values, it should be (height, width). /// \return Shared pointer to the current TensorOperation. std::shared_ptr SoftDvppDecodeRandomCropResizeJpeg( std::vector size, std::vector scale = {0.08, 1.0}, std::vector ratio = {3. / 4., 4. / 3.}, int32_t max_attempts = 10); /// \brief Function to create a SoftDvppDecodeResizeJpeg TensorOperation. /// \notes Tensor operation to decode and resize JPEG image using the simulation algorithm of Ascend series /// chip DVPP module. It is recommended to use this algorithm in the following scenarios: /// When training, the DVPP of the Ascend chip is not used, /// and the DVPP of the Ascend chip is used during inference, /// and the accuracy of inference is lower than the accuracy of training; /// and the input image size should be in range [32*32, 8192*8192]. /// The zoom-out and zoom-in multiples of the image length and width should in the range [1/32, 16]. /// Only images with an even resolution can be output. The output of odd resolution is not supported. /// \param[in] size A vector representing the output size of the resized image. /// If size is a single value, smaller edge of the image will be resized to this value with /// the same image aspect ratio. If size has 2 values, it should be (height, width). /// \return Shared pointer to the current TensorOperation. std::shared_ptr SoftDvppDecodeResizeJpeg(std::vector size); /// \brief Function to create a SwapRedBlue TensorOp /// \notes Swaps the red and blue channels in image /// \return Shared pointer to the current TensorOp std::shared_ptr SwapRedBlue(); /// \brief Function to create a UniformAugment TensorOperation. /// \notes Tensor operation to perform randomly selected augmentation. /// \param[in] transforms A vector of TensorOperation transforms. /// \param[in] num_ops An integer representing the number of OPs to be selected and applied. /// \return Shared pointer to the current TensorOperation. std::shared_ptr UniformAugment(std::vector> transforms, int32_t num_ops = 2); } // namespace vision } // namespace dataset } // namespace mindspore #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_