Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: f1542a90a3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f1542a90a3'
${ noResults }
mindspore/tests/ut/python/dataset/test_minddataset_sampler.py

264 lines
11 kB
Raw Blame History 

  1. # Copyright 2019 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. """

  16. This is the test module for mindrecord

  17. """

  18. import collections

  19. import json

  20. import os

  21. import re

  22. import string

  23. 

  24. import mindspore.dataset.transforms.vision.c_transforms as vision

  25. import numpy as np

  26. import pytest

  27. from mindspore.dataset.transforms.vision import Inter

  28. from mindspore import log as logger

  29. 

  30. import mindspore.dataset as ds

  31. from mindspore.mindrecord import FileWriter

  32. 

  33. FILES_NUM = 4

  34. CV_FILE_NAME = "../data/mindrecord/imagenet.mindrecord"

  35. CV_DIR_NAME = "../data/mindrecord/testImageNetData"

  36. 

  37. 

  38. @pytest.fixture

  39. def add_and_remove_cv_file():

  40.     """add/remove cv file"""

  41.     paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))

  42.              for x in range(FILES_NUM)]

  43.     for x in paths:

  44.         if os.path.exists("{}".format(x)):

  45.             os.remove("{}".format(x))

  46.         if os.path.exists("{}.db".format(x)):

  47.             os.remove("{}.db".format(x))

  48.     writer = FileWriter(CV_FILE_NAME, FILES_NUM)

  49.     data = get_data(CV_DIR_NAME, True)

  50.     cv_schema_json = {"id": {"type": "int32"},

  51.                       "file_name": {"type": "string"},

  52.                       "label": {"type": "int32"},

  53.                       "data": {"type": "bytes"}}

  54.     writer.add_schema(cv_schema_json, "img_schema")

  55.     writer.add_index(["file_name", "label"])

  56.     writer.write_raw_data(data)

  57.     writer.commit()

  58.     yield "yield_cv_data"

  59.     for x in paths:

  60.         os.remove("{}".format(x))

  61.         os.remove("{}.db".format(x))

  62. 

  63. 

  64. def test_cv_minddataset_pk_sample_basic(add_and_remove_cv_file):

  65.     """tutorial for cv minderdataset."""

  66.     columns_list = ["data", "file_name", "label"]

  67.     num_readers = 4

  68.     sampler = ds.PKSampler(2)

  69.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  70.                               sampler=sampler)

  71. 

  72.     assert data_set.get_dataset_size() == 6

  73.     num_iter = 0

  74.     for item in data_set.create_dict_iterator():

  75.         logger.info("-------------- cv reader basic: {} ------------------------".format(num_iter))

  76.         logger.info("-------------- item[file_name]: \

  77.                 {}------------------------".format("".join([chr(x) for x in item["file_name"]])))

  78.         logger.info("-------------- item[label]: {} ----------------------------".format(item["label"]))

  79.         num_iter += 1

  80. 

  81.     

  82. def test_cv_minddataset_pk_sample_shuffle(add_and_remove_cv_file):

  83.     """tutorial for cv minderdataset."""

  84.     columns_list = ["data", "file_name", "label"]

  85.     num_readers = 4

  86.     sampler = ds.PKSampler(3, None, True)

  87.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  88.                               sampler=sampler)

  89. 

  90.     assert data_set.get_dataset_size() == 9

  91.     num_iter = 0

  92.     for item in data_set.create_dict_iterator():

  93.         logger.info("-------------- cv reader basic: {} ------------------------".format(num_iter))

  94.         logger.info("-------------- item[file_name]: \

  95.                 {}------------------------".format("".join([chr(x) for x in item["file_name"]])))

  96.         logger.info("-------------- item[label]: {} ----------------------------".format(item["label"]))

  97.         num_iter += 1

  98. 

  99. 

  100. def test_cv_minddataset_pk_sample_out_of_range(add_and_remove_cv_file):

  101.     """tutorial for cv minderdataset."""

  102.     columns_list = ["data", "file_name", "label"]

  103.     num_readers = 4

  104.     sampler = ds.PKSampler(5, None, True)

  105.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  106.                               sampler=sampler)

  107.     assert data_set.get_dataset_size() == 15

  108.     num_iter = 0

  109.     for item in data_set.create_dict_iterator():

  110.         logger.info("-------------- cv reader basic: {} ------------------------".format(num_iter))

  111.         logger.info("-------------- item[file_name]: \

  112.                 {}------------------------".format("".join([chr(x) for x in item["file_name"]])))

  113.         logger.info("-------------- item[label]: {} ----------------------------".format(item["label"]))

  114.         num_iter += 1

  115. 

  116. 

  117. def test_cv_minddataset_subset_random_sample_basic(add_and_remove_cv_file):

  118.     """tutorial for cv minderdataset."""

  119.     columns_list = ["data", "file_name", "label"]

  120.     num_readers = 4

  121.     indices = [1, 2, 3, 5, 7]

  122.     sampler = ds.SubsetRandomSampler(indices)

  123.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  124.                               sampler=sampler)

  125.     assert data_set.get_dataset_size() == 5

  126.     num_iter = 0

  127.     for item in data_set.create_dict_iterator():

  128.         logger.info(

  129.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  130.         logger.info(

  131.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  132.         logger.info(

  133.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  134.         logger.info(

  135.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  136.         num_iter += 1

  137.     assert num_iter == 5

  138. 

  139. 

  140. def test_cv_minddataset_subset_random_sample_replica(add_and_remove_cv_file):

  141.     """tutorial for cv minderdataset."""

  142.     columns_list = ["data", "file_name", "label"]

  143.     num_readers = 4

  144.     indices = [1, 2, 2, 5, 7, 9]

  145.     sampler = ds.SubsetRandomSampler(indices)

  146.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  147.                               sampler=sampler)

  148.     assert data_set.get_dataset_size() == 6 

  149.     num_iter = 0

  150.     for item in data_set.create_dict_iterator():

  151.         logger.info(

  152.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  153.         logger.info(

  154.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  155.         logger.info(

  156.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  157.         logger.info(

  158.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  159.         num_iter += 1

  160.     assert num_iter == 6

  161. 

  162. 

  163. def test_cv_minddataset_subset_random_sample_empty(add_and_remove_cv_file):

  164.     """tutorial for cv minderdataset."""

  165.     columns_list = ["data", "file_name", "label"]

  166.     num_readers = 4

  167.     indices = []

  168.     sampler = ds.SubsetRandomSampler(indices)

  169.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  170.                               sampler=sampler)

  171.     assert data_set.get_dataset_size() == 0

  172.     num_iter = 0

  173.     for item in data_set.create_dict_iterator():

  174.         logger.info(

  175.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  176.         logger.info(

  177.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  178.         logger.info(

  179.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  180.         logger.info(

  181.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  182.         num_iter += 1

  183.     assert num_iter == 0

  184. 

  185. 

  186. def test_cv_minddataset_subset_random_sample_out_of_range(add_and_remove_cv_file):

  187.     """tutorial for cv minderdataset."""

  188.     columns_list = ["data", "file_name", "label"]

  189.     num_readers = 4

  190.     indices = [1, 2, 4, 11, 13]

  191.     sampler = ds.SubsetRandomSampler(indices)

  192.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  193.                               sampler=sampler)

  194.     assert data_set.get_dataset_size() == 5 

  195.     num_iter = 0

  196.     for item in data_set.create_dict_iterator():

  197.         logger.info(

  198.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  199.         logger.info(

  200.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  201.         logger.info(

  202.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  203.         logger.info(

  204.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  205.         num_iter += 1

  206.     assert num_iter == 5

  207. 

  208. 

  209. def test_cv_minddataset_subset_random_sample_negative(add_and_remove_cv_file):

  210.     """tutorial for cv minderdataset."""

  211.     columns_list = ["data", "file_name", "label"]

  212.     num_readers = 4

  213.     indices = [1, 2, 4, -1, -2]

  214.     sampler = ds.SubsetRandomSampler(indices)

  215.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  216.                               sampler=sampler)

  217.     assert data_set.get_dataset_size() == 5

  218.     num_iter = 0

  219.     for item in data_set.create_dict_iterator():

  220.         logger.info(

  221.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  222.         logger.info(

  223.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  224.         logger.info(

  225.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  226.         logger.info(

  227.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  228.         num_iter += 1

  229.     assert num_iter == 5

  230. 

  231. 

  232. def get_data(dir_name, sampler=False):

  233.     """

  234.     usage: get data from imagenet dataset

  235.     params:

  236.     dir_name: directory containing folder images and annotation information

  237. 

  238.     """

  239.     if not os.path.isdir(dir_name):

  240.         raise IOError("Directory {} not exists".format(dir_name))

  241.     img_dir = os.path.join(dir_name, "images")

  242.     if sampler:

  243.         ann_file = os.path.join(dir_name, "annotation_sampler.txt")

  244.     else:

  245.         ann_file = os.path.join(dir_name, "annotation.txt")

  246.     with open(ann_file, "r") as file_reader:

  247.         lines = file_reader.readlines()

  248. 

  249.     data_list = []

  250.     for i, line in enumerate(lines):

  251.         try:

  252.             filename, label = line.split(",")

  253.             label = label.strip("\n")

  254.             with open(os.path.join(img_dir, filename), "rb") as file_reader:

  255.                 img = file_reader.read()

  256.             data_json = {"id": i,

  257.                          "file_name": filename,

  258.                          "data": img,

  259.                          "label": int(label)}

  260.             data_list.append(data_json)

  261.         except FileNotFoundError:

  262.             continue

  263.     return data_list