Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: db80f4ff92
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'db80f4ff92'
${ noResults }
mindspore/tests/ut/python/dataset/test_datasets_sharding.py

164 lines
7.9 kB
Raw Blame History 

  1. # Copyright 2019 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. import mindspore.dataset as ds

  16. from mindspore import log as logger

  17. 

  18. def test_imagefolder_shardings(print_res=False):

  19.     image_folder_dir = "../data/dataset/testPK/data"

  20. 

  21.     def sharding_config(num_shards, shard_id, num_samples, shuffle, class_index, repeat_cnt=1):

  22.         data1 = ds.ImageFolderDatasetV2(image_folder_dir, num_samples=num_samples, num_shards=num_shards,

  23.                                       shard_id=shard_id,

  24.                                       shuffle=shuffle, class_indexing=class_index, decode=True)

  25.         data1 = data1.repeat(repeat_cnt)

  26.         res = []

  27.         for item in data1.create_dict_iterator():  # each data is a dictionary

  28.             res.append(item["label"].item())

  29.         if (print_res):

  30.             logger.info("labels of dataset: {}".format(res))

  31.         return res

  32. 

  33.     # total 44 rows in dataset

  34.     assert (sharding_config(4, 0, 5, False, dict()) == [0, 0, 0, 1, 1])  # 5 rows

  35.     assert (sharding_config(4, 0, 12, False, dict()) == [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3])  # 11 rows

  36.     assert (sharding_config(4, 3, None, False, dict()) == [0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])  # 11 rows

  37.     # total 22 in dataset rows because of class indexing which takes only 2 folders

  38.     assert (len(sharding_config(4, 0, None, True, {"class1": 111, "class2": 999})) == 6)

  39.     assert (len(sharding_config(4, 2, 3, True, {"class1": 111, "class2": 999})) == 3)

  40.     # test with repeat

  41.     assert (sharding_config(4, 0, 12, False, dict(), 3) == [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3] * 3)

  42.     assert (sharding_config(4, 0, 5, False, dict(), 5) == [0, 0, 0, 1, 1] * 5)

  43.     assert (len(sharding_config(5, 1, None, True, {"class1": 111, "class2": 999}, 4)) == 20)

  44. 

  45. 

  46. def test_manifest_shardings(print_res=False):

  47.     manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"

  48. 

  49.     def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1):

  50.         data1 = ds.ManifestDataset(manifest_file, num_samples=num_samples, num_shards=num_shards, shard_id=shard_id,

  51.                                  shuffle=shuffle, decode=True)

  52.         data1 = data1.repeat(repeat_cnt)

  53.         res = []

  54.         for item in data1.create_dict_iterator():  # each data is a dictionary

  55.             res.append(item["label"].item())

  56.         if print_res:

  57.             logger.info("labels of dataset: {}".format(res))

  58.         return res

  59. 

  60.     # 5 train images in total

  61.     sharding_config(2, 0, None, False)

  62.     assert (sharding_config(2, 0, None, False) == [0, 1, 1])

  63.     assert (sharding_config(2, 1, None, False) == [0, 0, 0])

  64.     assert (sharding_config(2, 0, 2, False) == [0, 1])

  65.     assert (sharding_config(2, 1, 2, False) == [0, 0])

  66.     # with repeat

  67.     assert (sharding_config(2, 1, None, False, 3) == [0, 0, 0] * 3)

  68.     assert (sharding_config(2, 0, 2, False, 5) == [0, 1] * 5)

  69. 

  70. 

  71. def test_voc_shardings(print_res=False):

  72.     voc_dir = "../data/dataset/testVOC2012"

  73. 

  74.     def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1):

  75.         sampler = ds.DistributedSampler(num_shards, shard_id, shuffle=shuffle)

  76.         data1 = ds.VOCDataset(voc_dir, decode=True, sampler=sampler, num_samples=num_samples)

  77.         data1 = data1.repeat(repeat_cnt)

  78.         res = []

  79.         for item in data1.create_dict_iterator():  # each data is a dictionary

  80.             res.append(item["image"].shape[0])

  81.         if print_res:

  82.             logger.info("labels of dataset: {}".format(res))

  83.         return res

  84. 

  85.     # 10 images in total, always decode to get the shape

  86.     # first dim of all 10 images [2268,2268,2268,2268,642,607,561,596,612,2268]

  87.     # 3 shard_workers, 0th worker will get 0-th, 3nd, 6th and 9th image

  88.     assert (sharding_config(3, 0, None, False, 2) == [2268, 2268, 561, 2268] * 2)

  89.     # 3 shard_workers, 1st worker will get 1-st, 4nd, 7th and 0th image, the last one goes back bc of rounding up

  90.     assert (sharding_config(3, 1, 5, False, 3) == [2268, 642, 596, 2268] * 3)

  91.     # 3 shard_workers, 2nd worker will get 2nd, 5th, 8th and 11th (which is 1st)

  92.     # then takes the first 2 bc num_samples = 2

  93.     assert (sharding_config(3, 2, 2, False, 4) == [2268, 607] * 4)

  94.     # test that each epoch, each shard_worker returns a different sample

  95.     assert (len(sharding_config(2, 0, None, True, 1)) == 5)

  96.     assert (len(set(sharding_config(11, 0, None, True, 10))) > 1)

  97. 

  98. 

  99. def test_cifar10_shardings(print_res=False):

  100.     cifar10_dir = "../data/dataset/testCifar10Data"

  101. 

  102.     def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1):

  103.         data1 = ds.Cifar10Dataset(cifar10_dir, num_shards=num_shards, shard_id=shard_id, num_samples=num_samples,

  104.                                 shuffle=shuffle)

  105.         data1 = data1.repeat(repeat_cnt)

  106.         res = []

  107.         for item in data1.create_dict_iterator():  # each data is a dictionary

  108.             res.append(item["label"].item())

  109.         if print_res:

  110.             logger.info("labels of dataset: {}".format(res))

  111.         return res

  112. 

  113.     # 60000 rows in total. CIFAR reads everything in memory which would make each test case very slow

  114.     # therefore, only 2 test cases for now.

  115.     assert (sharding_config(10000, 9999, 7, False, 1) == [9])

  116.     assert (sharding_config(10000, 0, 4, False, 3) == [0, 0, 0])

  117. 

  118. 

  119. def test_cifar100_shardings(print_res=False):

  120.     cifar100_dir = "../data/dataset/testCifar100Data"

  121. 

  122.     def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1):

  123.         data1 = ds.Cifar100Dataset(cifar100_dir, num_shards=num_shards, shard_id=shard_id, num_samples=num_samples,

  124.                                  shuffle=shuffle)

  125.         data1 = data1.repeat(repeat_cnt)

  126.         res = []

  127.         for item in data1.create_dict_iterator():  # each data is a dictionary

  128.             res.append(item["coarse_label"].item())

  129.         if print_res:

  130.             logger.info("labels of dataset: {}".format(res))

  131.         return res

  132. 

  133.     # 10000 rows in total in test.bin CIFAR100 file

  134.     assert (sharding_config(1000, 999, 7, False, 2) == [1, 18, 10, 17, 5, 0, 15] * 2)

  135.     assert (sharding_config(1000, 0, None, False) == [10, 16, 2, 11, 10, 17, 11, 14, 13, 3])

  136. 

  137. 

  138. def test_mnist_shardings(print_res=False):

  139.     mnist_dir = "../data/dataset/testMnistData"

  140. 

  141.     def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1):

  142.         data1 = ds.MnistDataset(mnist_dir, num_shards=num_shards, shard_id=shard_id, num_samples=num_samples,

  143.                               shuffle=shuffle)

  144.         data1 = data1.repeat(repeat_cnt)

  145.         res = []

  146.         for item in data1.create_dict_iterator():  # each data is a dictionary

  147.             res.append(item["label"].item())

  148.         if print_res:

  149.             logger.info("labels of dataset: {}".format(res))

  150.         return res

  151. 

  152.     # 70K rows in total , divide across 10K hosts, each host has 7 images

  153.     assert sharding_config(10000, 0, num_samples=5, shuffle=False, repeat_cnt=3) == [0, 0, 0]

  154.     assert sharding_config(10000, 9999, num_samples=None, shuffle=False, repeat_cnt=1) == [9]

  155. 

  156. 

  157. if __name__ == '__main__':

  158.     test_imagefolder_shardings(True)

  159.     test_manifest_shardings(True)

  160.     test_voc_shardings(True)

  161.     test_cifar10_shardings(True)

  162.     test_cifar100_shardings(True)

  163.     test_mnist_shardings(True)