Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 117b35eb49
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '117b35eb49'
${ noResults }
mindspore/tests/ut/python/dataset/test_autocontrast.py

353 lines
17 kB
Raw Blame History 

  1. # Copyright 2020 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. """

  16. Testing AutoContrast op in DE

  17. """

  18. import numpy as np

  19. import mindspore.dataset as ds

  20. import mindspore.dataset.transforms.py_transforms

  21. import mindspore.dataset.vision.py_transforms as F

  22. import mindspore.dataset.vision.c_transforms as C

  23. from mindspore import log as logger

  24. from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5

  25. 

  26. DATA_DIR = "../data/dataset/testImageNetData/train/"

  27. MNIST_DATA_DIR = "../data/dataset/testMnistData"

  28. 

  29. GENERATE_GOLDEN = False

  30. 

  31. 

  32. def test_auto_contrast_py(plot=False):

  33.     """

  34.     Test AutoContrast

  35.     """

  36.     logger.info("Test AutoContrast Python Op")

  37. 

  38.     # Original Images

  39.     data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  40. 

  41.     transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  42.                                                                               F.Resize((224, 224)),

  43.                                                                               F.ToTensor()])

  44. 

  45.     ds_original = data_set.map(operations=transforms_original, input_columns="image")

  46. 

  47.     ds_original = ds_original.batch(512)

  48. 

  49.     for idx, (image, _) in enumerate(ds_original):

  50.         if idx == 0:

  51.             images_original = np.transpose(image.asnumpy(), (0, 2, 3, 1))

  52.         else:

  53.             images_original = np.append(images_original,

  54.                                         np.transpose(image.asnumpy(), (0, 2, 3, 1)),

  55.                                         axis=0)

  56. 

  57.             # AutoContrast Images

  58.     data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  59. 

  60.     transforms_auto_contrast = \

  61.         mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  62.                                                             F.Resize((224, 224)),

  63.                                                             F.AutoContrast(cutoff=10.0, ignore=[10, 20]),

  64.                                                             F.ToTensor()])

  65. 

  66.     ds_auto_contrast = data_set.map(operations=transforms_auto_contrast, input_columns="image")

  67. 

  68.     ds_auto_contrast = ds_auto_contrast.batch(512)

  69. 

  70.     for idx, (image, _) in enumerate(ds_auto_contrast):

  71.         if idx == 0:

  72.             images_auto_contrast = np.transpose(image.asnumpy(), (0, 2, 3, 1))

  73.         else:

  74.             images_auto_contrast = np.append(images_auto_contrast,

  75.                                              np.transpose(image.asnumpy(), (0, 2, 3, 1)),

  76.                                              axis=0)

  77. 

  78.     num_samples = images_original.shape[0]

  79.     mse = np.zeros(num_samples)

  80.     for i in range(num_samples):

  81.         mse[i] = diff_mse(images_auto_contrast[i], images_original[i])

  82.     logger.info("MSE= {}".format(str(np.mean(mse))))

  83. 

  84.     # Compare with expected md5 from images

  85.     filename = "autocontrast_01_result_py.npz"

  86.     save_and_check_md5(ds_auto_contrast, filename, generate_golden=GENERATE_GOLDEN)

  87. 

  88.     if plot:

  89.         visualize_list(images_original, images_auto_contrast)

  90. 

  91. 

  92. def test_auto_contrast_c(plot=False):

  93.     """

  94.     Test AutoContrast C Op

  95.     """

  96.     logger.info("Test AutoContrast C Op")

  97. 

  98.     # AutoContrast Images

  99.     data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  100.     data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224))], input_columns=["image"])

  101.     python_op = F.AutoContrast(cutoff=10.0, ignore=[10, 20])

  102.     c_op = C.AutoContrast(cutoff=10.0, ignore=[10, 20])

  103.     transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)),

  104.                                                                         python_op,

  105.                                                                         np.array])

  106. 

  107.     ds_auto_contrast_py = data_set.map(operations=transforms_op, input_columns="image")

  108. 

  109.     ds_auto_contrast_py = ds_auto_contrast_py.batch(512)

  110. 

  111.     for idx, (image, _) in enumerate(ds_auto_contrast_py):

  112.         if idx == 0:

  113.             images_auto_contrast_py = image.asnumpy()

  114.         else:

  115.             images_auto_contrast_py = np.append(images_auto_contrast_py,

  116.                                                 image.asnumpy(),

  117.                                                 axis=0)

  118. 

  119.     data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  120.     data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224))], input_columns=["image"])

  121. 

  122.     ds_auto_contrast_c = data_set.map(operations=c_op, input_columns="image")

  123. 

  124.     ds_auto_contrast_c = ds_auto_contrast_c.batch(512)

  125. 

  126.     for idx, (image, _) in enumerate(ds_auto_contrast_c):

  127.         if idx == 0:

  128.             images_auto_contrast_c = image.asnumpy()

  129.         else:

  130.             images_auto_contrast_c = np.append(images_auto_contrast_c,

  131.                                                image.asnumpy(),

  132.                                                axis=0)

  133. 

  134.     num_samples = images_auto_contrast_c.shape[0]

  135.     mse = np.zeros(num_samples)

  136.     for i in range(num_samples):

  137.         mse[i] = diff_mse(images_auto_contrast_c[i], images_auto_contrast_py[i])

  138.     logger.info("MSE= {}".format(str(np.mean(mse))))

  139.     np.testing.assert_equal(np.mean(mse), 0.0)

  140. 

  141.     # Compare with expected md5 from images

  142.     filename = "autocontrast_01_result_c.npz"

  143.     save_and_check_md5(ds_auto_contrast_c, filename, generate_golden=GENERATE_GOLDEN)

  144. 

  145.     if plot:

  146.         visualize_list(images_auto_contrast_c, images_auto_contrast_py, visualize_mode=2)

  147. 

  148. 

  149. def test_auto_contrast_one_channel_c(plot=False):

  150.     """

  151.     Test AutoContrast C op with one channel

  152.     """

  153.     logger.info("Test AutoContrast C Op With One Channel Images")

  154. 

  155.     # AutoContrast Images

  156.     data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  157.     data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224))], input_columns=["image"])

  158.     python_op = F.AutoContrast()

  159.     c_op = C.AutoContrast()

  160.     # not using F.ToTensor() since it converts to floats

  161.     transforms_op = mindspore.dataset.transforms.py_transforms.Compose(

  162.         [lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),

  163.          F.ToPIL(),

  164.          python_op,

  165.          np.array])

  166. 

  167.     ds_auto_contrast_py = data_set.map(operations=transforms_op, input_columns="image")

  168. 

  169.     ds_auto_contrast_py = ds_auto_contrast_py.batch(512)

  170. 

  171.     for idx, (image, _) in enumerate(ds_auto_contrast_py):

  172.         if idx == 0:

  173.             images_auto_contrast_py = image.asnumpy()

  174.         else:

  175.             images_auto_contrast_py = np.append(images_auto_contrast_py,

  176.                                                 image.asnumpy(),

  177.                                                 axis=0)

  178. 

  179.     data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  180.     data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])],

  181.                             input_columns=["image"])

  182. 

  183.     ds_auto_contrast_c = data_set.map(operations=c_op, input_columns="image")

  184. 

  185.     ds_auto_contrast_c = ds_auto_contrast_c.batch(512)

  186. 

  187.     for idx, (image, _) in enumerate(ds_auto_contrast_c):

  188.         if idx == 0:

  189.             images_auto_contrast_c = image.asnumpy()

  190.         else:

  191.             images_auto_contrast_c = np.append(images_auto_contrast_c,

  192.                                                image.asnumpy(),

  193.                                                axis=0)

  194. 

  195.     num_samples = images_auto_contrast_c.shape[0]

  196.     mse = np.zeros(num_samples)

  197.     for i in range(num_samples):

  198.         mse[i] = diff_mse(np.squeeze(images_auto_contrast_c[i]), images_auto_contrast_py[i])

  199.     logger.info("MSE= {}".format(str(np.mean(mse))))

  200.     np.testing.assert_equal(np.mean(mse), 0.0)

  201. 

  202.     if plot:

  203.         visualize_list(images_auto_contrast_c, images_auto_contrast_py, visualize_mode=2)

  204. 

  205. 

  206. def test_auto_contrast_mnist_c(plot=False):

  207.     """

  208.     Test AutoContrast C op with MNIST dataset (Grayscale images)

  209.     """

  210.     logger.info("Test AutoContrast C Op With MNIST Images")

  211.     data_set = ds.MnistDataset(dataset_dir=MNIST_DATA_DIR, num_samples=2, shuffle=False)

  212.     ds_auto_contrast_c = data_set.map(operations=C.AutoContrast(cutoff=1, ignore=(0, 255)), input_columns="image")

  213.     ds_orig = ds.MnistDataset(dataset_dir=MNIST_DATA_DIR, num_samples=2, shuffle=False)

  214. 

  215.     images = []

  216.     images_trans = []

  217.     labels = []

  218.     for _, (data_orig, data_trans) in enumerate(zip(ds_orig, ds_auto_contrast_c)):

  219.         image_orig, label_orig = data_orig

  220.         image_trans, _ = data_trans

  221.         images.append(image_orig.asnumpy())

  222.         labels.append(label_orig.asnumpy())

  223.         images_trans.append(image_trans.asnumpy())

  224. 

  225.     # Compare with expected md5 from images

  226.     filename = "autocontrast_mnist_result_c.npz"

  227.     save_and_check_md5(ds_auto_contrast_c, filename, generate_golden=GENERATE_GOLDEN)

  228. 

  229.     if plot:

  230.         visualize_one_channel_dataset(images, images_trans, labels)

  231. 

  232. 

  233. def test_auto_contrast_invalid_ignore_param_c():

  234.     """

  235.     Test AutoContrast C Op with invalid ignore parameter

  236.     """

  237.     logger.info("Test AutoContrast C Op with invalid ignore parameter")

  238.     try:

  239.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  240.         data_set = data_set.map(operations=[C.Decode(),

  241.                                             C.Resize((224, 224)),

  242.                                             lambda img: np.array(img[:, :, 0])], input_columns=["image"])

  243.         # invalid ignore

  244.         data_set = data_set.map(operations=C.AutoContrast(ignore=255.5), input_columns="image")

  245.     except TypeError as error:

  246.         logger.info("Got an exception in DE: {}".format(str(error)))

  247.         assert "Argument ignore with value 255.5 is not of type" in str(error)

  248.     try:

  249.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  250.         data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)),

  251.                                             lambda img: np.array(img[:, :, 0])], input_columns=["image"])

  252.         # invalid ignore

  253.         data_set = data_set.map(operations=C.AutoContrast(ignore=(10, 100)), input_columns="image")

  254.     except TypeError as error:

  255.         logger.info("Got an exception in DE: {}".format(str(error)))

  256.         assert "Argument ignore with value (10,100) is not of type" in str(error)

  257. 

  258. 

  259. def test_auto_contrast_invalid_cutoff_param_c():

  260.     """

  261.     Test AutoContrast C Op with invalid cutoff parameter

  262.     """

  263.     logger.info("Test AutoContrast C Op with invalid cutoff parameter")

  264.     try:

  265.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  266.         data_set = data_set.map(operations=[C.Decode(),

  267.                                             C.Resize((224, 224)),

  268.                                             lambda img: np.array(img[:, :, 0])], input_columns=["image"])

  269.         # invalid ignore

  270.         data_set = data_set.map(operations=C.AutoContrast(cutoff=-10.0), input_columns="image")

  271.     except ValueError as error:

  272.         logger.info("Got an exception in DE: {}".format(str(error)))

  273.         assert "Input cutoff is not within the required interval of [0, 50)." in str(error)

  274.     try:

  275.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  276.         data_set = data_set.map(operations=[C.Decode(),

  277.                                             C.Resize((224, 224)),

  278.                                             lambda img: np.array(img[:, :, 0])], input_columns=["image"])

  279.         # invalid ignore

  280.         data_set = data_set.map(operations=C.AutoContrast(cutoff=120.0), input_columns="image")

  281.     except ValueError as error:

  282.         logger.info("Got an exception in DE: {}".format(str(error)))

  283.         assert "Input cutoff is not within the required interval of [0, 50)." in str(error)

  284. 

  285. 

  286. def test_auto_contrast_invalid_ignore_param_py():

  287.     """

  288.     Test AutoContrast python Op with invalid ignore parameter

  289.     """

  290.     logger.info("Test AutoContrast python Op with invalid ignore parameter")

  291.     try:

  292.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  293.         data_set = data_set.map(operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  294.                                                                                                 F.Resize((224, 224)),

  295.                                                                                                 F.AutoContrast(

  296.                                                                                                     ignore=255.5),

  297.                                                                                                 F.ToTensor()])],

  298.                                 input_columns=["image"])

  299.     except TypeError as error:

  300.         logger.info("Got an exception in DE: {}".format(str(error)))

  301.         assert "Argument ignore with value 255.5 is not of type" in str(error)

  302.     try:

  303.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  304.         data_set = data_set.map(operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  305.                                                                                                 F.Resize((224, 224)),

  306.                                                                                                 F.AutoContrast(

  307.                                                                                                     ignore=(10, 100)),

  308.                                                                                                 F.ToTensor()])],

  309.                                 input_columns=["image"])

  310.     except TypeError as error:

  311.         logger.info("Got an exception in DE: {}".format(str(error)))

  312.         assert "Argument ignore with value (10,100) is not of type" in str(error)

  313. 

  314. 

  315. def test_auto_contrast_invalid_cutoff_param_py():

  316.     """

  317.     Test AutoContrast python Op with invalid cutoff parameter

  318.     """

  319.     logger.info("Test AutoContrast python Op with invalid cutoff parameter")

  320.     try:

  321.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  322.         data_set = data_set.map(operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  323.                                                                                                 F.Resize((224, 224)),

  324.                                                                                                 F.AutoContrast(

  325.                                                                                                     cutoff=-10.0),

  326.                                                                                                 F.ToTensor()])],

  327.                                 input_columns=["image"])

  328.     except ValueError as error:

  329.         logger.info("Got an exception in DE: {}".format(str(error)))

  330.         assert "Input cutoff is not within the required interval of [0, 50)." in str(error)

  331.     try:

  332.         data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)

  333.         data_set = data_set.map(

  334.             operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),

  335.                                                                             F.Resize((224, 224)),

  336.                                                                             F.AutoContrast(cutoff=120.0),

  337.                                                                             F.ToTensor()])],

  338.             input_columns=["image"])

  339.     except ValueError as error:

  340.         logger.info("Got an exception in DE: {}".format(str(error)))

  341.         assert "Input cutoff is not within the required interval of [0, 50)." in str(error)

  342. 

  343. 

  344. if __name__ == "__main__":

  345.     test_auto_contrast_py(plot=True)

  346.     test_auto_contrast_c(plot=True)

  347.     test_auto_contrast_one_channel_c(plot=True)

  348.     test_auto_contrast_mnist_c(plot=True)

  349.     test_auto_contrast_invalid_ignore_param_c()

  350.     test_auto_contrast_invalid_ignore_param_py()

  351.     test_auto_contrast_invalid_cutoff_param_c()

  352.     test_auto_contrast_invalid_cutoff_param_py()