Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 31fed1a2f6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '31fed1a2f6'
${ noResults }
mindspore/tests/ut/python/dataset/test_random_sharpness.py

339 lines
13 kB
Raw Blame History 

  1. # Copyright 2020 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. """

  16. Testing RandomSharpness op in DE

  17. """

  18. import numpy as np

  19. import mindspore.dataset as ds

  20. import mindspore.dataset.transforms.py_transforms

  21. import mindspore.dataset.vision.py_transforms as F

  22. import mindspore.dataset.vision.c_transforms as C

  23. from mindspore import log as logger

  24. from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5, \

  25.     config_get_set_seed, config_get_set_num_parallel_workers

  26. 

  27. DATA_DIR = "../data/dataset/testImageNetData/train/"

  28. MNIST_DATA_DIR = "../data/dataset/testMnistData"

  29. 

  30. GENERATE_GOLDEN = False

  31. 

  32. 

  33. def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False):

  34.     """

  35.     Test RandomSharpness python op

  36.     """

  37.     logger.info("Test RandomSharpness python op")

  38. 

  39.     # Original Images

  40.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  41. 

  42.     transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  43.                                                                               F.Resize((224, 224)),

  44.                                                                               F.ToTensor()])

  45. 

  46.     ds_original = data.map(operations=transforms_original, input_columns="image")

  47. 

  48.     ds_original = ds_original.batch(512)

  49. 

  50.     for idx, (image, _) in enumerate(ds_original.create_tuple_iterator(output_numpy=True)):

  51.         if idx == 0:

  52.             images_original = np.transpose(image, (0, 2, 3, 1))

  53.         else:

  54.             images_original = np.append(images_original,

  55.                                         np.transpose(image, (0, 2, 3, 1)),

  56.                                         axis=0)

  57. 

  58.     # Random Sharpness Adjusted Images

  59.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  60. 

  61.     py_op = F.RandomSharpness()

  62.     if degrees is not None:

  63.         py_op = F.RandomSharpness(degrees)

  64. 

  65.     transforms_random_sharpness = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  66.                                                                                       F.Resize((224, 224)),

  67.                                                                                       py_op,

  68.                                                                                       F.ToTensor()])

  69. 

  70.     ds_random_sharpness = data.map(operations=transforms_random_sharpness, input_columns="image")

  71. 

  72.     ds_random_sharpness = ds_random_sharpness.batch(512)

  73. 

  74.     for idx, (image, _) in enumerate(ds_random_sharpness.create_tuple_iterator(output_numpy=True)):

  75.         if idx == 0:

  76.             images_random_sharpness = np.transpose(image, (0, 2, 3, 1))

  77.         else:

  78.             images_random_sharpness = np.append(images_random_sharpness,

  79.                                                 np.transpose(image, (0, 2, 3, 1)),

  80.                                                 axis=0)

  81. 

  82.     num_samples = images_original.shape[0]

  83.     mse = np.zeros(num_samples)

  84.     for i in range(num_samples):

  85.         mse[i] = diff_mse(images_random_sharpness[i], images_original[i])

  86. 

  87.     logger.info("MSE= {}".format(str(np.mean(mse))))

  88. 

  89.     if plot:

  90.         visualize_list(images_original, images_random_sharpness)

  91. 

  92. 

  93. def test_random_sharpness_py_md5():

  94.     """

  95.     Test RandomSharpness python op with md5 comparison

  96.     """

  97.     logger.info("Test RandomSharpness python op with md5 comparison")

  98.     original_seed = config_get_set_seed(5)

  99.     original_num_parallel_workers = config_get_set_num_parallel_workers(1)

  100. 

  101.     # define map operations

  102.     transforms = [

  103.         F.Decode(),

  104.         F.RandomSharpness((20.0, 25.0)),

  105.         F.ToTensor()

  106.     ]

  107.     transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)

  108. 

  109.     #  Generate dataset

  110.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  111.     data = data.map(operations=transform, input_columns=["image"])

  112. 

  113.     # check results with md5 comparison

  114.     filename = "random_sharpness_py_01_result.npz"

  115.     save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)

  116. 

  117.     # Restore configuration

  118.     ds.config.set_seed(original_seed)

  119.     ds.config.set_num_parallel_workers(original_num_parallel_workers)

  120. 

  121. 

  122. def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False):

  123.     """

  124.     Test RandomSharpness cpp op

  125.     """

  126.     print(degrees)

  127.     logger.info("Test RandomSharpness cpp op")

  128. 

  129.     # Original Images

  130.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  131. 

  132.     transforms_original = [C.Decode(),

  133.                            C.Resize((224, 224))]

  134. 

  135.     ds_original = data.map(operations=transforms_original, input_columns="image")

  136. 

  137.     ds_original = ds_original.batch(512)

  138. 

  139.     for idx, (image, _) in enumerate(ds_original.create_tuple_iterator(output_numpy=True)):

  140.         if idx == 0:

  141.             images_original = image

  142.         else:

  143.             images_original = np.append(images_original,

  144.                                         image,

  145.                                         axis=0)

  146. 

  147.             # Random Sharpness Adjusted Images

  148.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  149. 

  150.     c_op = C.RandomSharpness()

  151.     if degrees is not None:

  152.         c_op = C.RandomSharpness(degrees)

  153. 

  154.     transforms_random_sharpness = [C.Decode(),

  155.                                    C.Resize((224, 224)),

  156.                                    c_op]

  157. 

  158.     ds_random_sharpness = data.map(operations=transforms_random_sharpness, input_columns="image")

  159. 

  160.     ds_random_sharpness = ds_random_sharpness.batch(512)

  161. 

  162.     for idx, (image, _) in enumerate(ds_random_sharpness.create_tuple_iterator(output_numpy=True)):

  163.         if idx == 0:

  164.             images_random_sharpness = image

  165.         else:

  166.             images_random_sharpness = np.append(images_random_sharpness,

  167.                                                 image,

  168.                                                 axis=0)

  169. 

  170.     num_samples = images_original.shape[0]

  171.     mse = np.zeros(num_samples)

  172.     for i in range(num_samples):

  173.         mse[i] = diff_mse(images_random_sharpness[i], images_original[i])

  174. 

  175.     logger.info("MSE= {}".format(str(np.mean(mse))))

  176. 

  177.     if plot:

  178.         visualize_list(images_original, images_random_sharpness)

  179. 

  180. 

  181. def test_random_sharpness_c_md5():

  182.     """

  183.     Test RandomSharpness cpp op with md5 comparison

  184.     """

  185.     logger.info("Test RandomSharpness cpp op with md5 comparison")

  186.     original_seed = config_get_set_seed(200)

  187.     original_num_parallel_workers = config_get_set_num_parallel_workers(1)

  188. 

  189.     # define map operations

  190.     transforms = [

  191.         C.Decode(),

  192.         C.RandomSharpness((10.0, 15.0))

  193.     ]

  194. 

  195.     #  Generate dataset

  196.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  197.     data = data.map(operations=transforms, input_columns=["image"])

  198. 

  199.     # check results with md5 comparison

  200.     filename = "random_sharpness_cpp_01_result.npz"

  201.     save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)

  202. 

  203.     # Restore configuration

  204.     ds.config.set_seed(original_seed)

  205.     ds.config.set_num_parallel_workers(original_num_parallel_workers)

  206. 

  207. 

  208. def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False):

  209.     """

  210.     Test Random Sharpness C and python Op

  211.     """

  212.     logger.info("Test RandomSharpness C and python Op")

  213. 

  214.     # RandomSharpness Images

  215.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  216.     data = data.map(operations=[C.Decode(), C.Resize((200, 300))], input_columns=["image"])

  217. 

  218.     python_op = F.RandomSharpness(degrees)

  219.     c_op = C.RandomSharpness(degrees)

  220. 

  221.     transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)),

  222.                                                                         python_op,

  223.                                                                         np.array])

  224. 

  225.     ds_random_sharpness_py = data.map(operations=transforms_op, input_columns="image")

  226. 

  227.     ds_random_sharpness_py = ds_random_sharpness_py.batch(512)

  228. 

  229.     for idx, (image, _) in enumerate(ds_random_sharpness_py.create_tuple_iterator(output_numpy=True)):

  230.         if idx == 0:

  231.             images_random_sharpness_py = image

  232. 

  233.         else:

  234.             images_random_sharpness_py = np.append(images_random_sharpness_py,

  235.                                                    image,

  236.                                                    axis=0)

  237. 

  238.     data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  239.     data = data.map(operations=[C.Decode(), C.Resize((200, 300))], input_columns=["image"])

  240. 

  241.     ds_images_random_sharpness_c = data.map(operations=c_op, input_columns="image")

  242. 

  243.     ds_images_random_sharpness_c = ds_images_random_sharpness_c.batch(512)

  244. 

  245.     for idx, (image, _) in enumerate(ds_images_random_sharpness_c.create_tuple_iterator(output_numpy=True)):

  246.         if idx == 0:

  247.             images_random_sharpness_c = image

  248. 

  249.         else:

  250.             images_random_sharpness_c = np.append(images_random_sharpness_c,

  251.                                                   image,

  252.                                                   axis=0)

  253. 

  254.     num_samples = images_random_sharpness_c.shape[0]

  255.     mse = np.zeros(num_samples)

  256.     for i in range(num_samples):

  257.         mse[i] = diff_mse(images_random_sharpness_c[i], images_random_sharpness_py[i])

  258.     logger.info("MSE= {}".format(str(np.mean(mse))))

  259.     if plot:

  260.         visualize_list(images_random_sharpness_c, images_random_sharpness_py, visualize_mode=2)

  261. 

  262. 

  263. def test_random_sharpness_one_channel_c(degrees=(1.4, 1.4), plot=False):

  264.     """

  265.     Test Random Sharpness cpp op with one channel

  266.     """

  267.     logger.info("Test RandomSharpness C Op With MNIST Dataset (Grayscale images)")

  268. 

  269.     c_op = C.RandomSharpness()

  270.     if degrees is not None:

  271.         c_op = C.RandomSharpness(degrees)

  272.     # RandomSharpness Images

  273.     data = ds.MnistDataset(dataset_dir=MNIST_DATA_DIR, num_samples=2, shuffle=False)

  274.     ds_random_sharpness_c = data.map(operations=c_op, input_columns="image")

  275.     # Original images

  276.     data = ds.MnistDataset(dataset_dir=MNIST_DATA_DIR, num_samples=2, shuffle=False)

  277. 

  278.     images = []

  279.     images_trans = []

  280.     labels = []

  281.     for _, (data_orig, data_trans) in enumerate(zip(data, ds_random_sharpness_c)):

  282.         image_orig, label_orig = data_orig

  283.         image_trans, _ = data_trans

  284.         images.append(image_orig.asnumpy())

  285.         labels.append(label_orig.asnumpy())

  286.         images_trans.append(image_trans.asnumpy())

  287. 

  288.     if plot:

  289.         visualize_one_channel_dataset(images, images_trans, labels)

  290. 

  291. 

  292. def test_random_sharpness_invalid_params():

  293.     """

  294.     Test RandomSharpness with invalid input parameters.

  295.     """

  296.     logger.info("Test RandomSharpness with invalid input parameters.")

  297.     try:

  298.         data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  299.         data = data.map(operations=[C.Decode(), C.Resize((224, 224)),

  300.                                     C.RandomSharpness(10)], input_columns=["image"])

  301.     except TypeError as error:

  302.         logger.info("Got an exception in DE: {}".format(str(error)))

  303.         assert "tuple" in str(error)

  304. 

  305.     try:

  306.         data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  307.         data = data.map(operations=[C.Decode(), C.Resize((224, 224)),

  308.                                     C.RandomSharpness((-10, 10))], input_columns=["image"])

  309.     except ValueError as error:

  310.         logger.info("Got an exception in DE: {}".format(str(error)))

  311.         assert "interval" in str(error)

  312. 

  313.     try:

  314.         data = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  315.         data = data.map(operations=[C.Decode(), C.Resize((224, 224)),

  316.                                     C.RandomSharpness((10, 5))], input_columns=["image"])

  317.     except ValueError as error:

  318.         logger.info("Got an exception in DE: {}".format(str(error)))

  319.         assert "(min,max)" in str(error)

  320. 

  321. 

  322. if __name__ == "__main__":

  323.     test_random_sharpness_py(plot=True)

  324.     test_random_sharpness_py(None, plot=True)  # Test with default values

  325.     test_random_sharpness_py(degrees=(20.0, 25.0),

  326.                              plot=True)  # Test with degree values that show more obvious transformation

  327.     test_random_sharpness_py_md5()

  328.     test_random_sharpness_c(plot=True)

  329.     test_random_sharpness_c(None, plot=True)  # test with default values

  330.     test_random_sharpness_c(degrees=[10, 15],

  331.                             plot=True)  # Test with degrees values that show more obvious transformation

  332.     test_random_sharpness_c_md5()

  333.     test_random_sharpness_c_py(degrees=[1.5, 1.5], plot=True)

  334.     test_random_sharpness_c_py(degrees=[1, 1], plot=True)

  335.     test_random_sharpness_c_py(degrees=[10, 10], plot=True)

  336.     test_random_sharpness_one_channel_c(degrees=[1.7, 1.7], plot=True)

  337.     test_random_sharpness_one_channel_c(degrees=None, plot=True)  # Test with default values

  338.     test_random_sharpness_invalid_params()