Browse Source

!12555 Create C++ API classes for Samplers

From: @mhmotallebi
Reviewed-by: 
Signed-off-by:
tags/v1.2.0-rc1
mindspore-ci-bot Gitee 4 years ago
parent
commit
f1c7aeaa6b
29 changed files with 1498 additions and 522 deletions
  1. +391
    -34
      mindspore/ccsrc/minddata/dataset/api/datasets.cc
  2. +50
    -60
      mindspore/ccsrc/minddata/dataset/api/samplers.cc
  3. +5
    -5
      mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc
  4. +398
    -37
      mindspore/ccsrc/minddata/dataset/include/datasets.h
  5. +206
    -57
      mindspore/ccsrc/minddata/dataset/include/samplers.h
  6. +16
    -16
      tests/ut/cpp/dataset/c_api_cache_test.cc
  7. +3
    -3
      tests/ut/cpp/dataset/c_api_dataset_album_test.cc
  8. +9
    -9
      tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc
  9. +6
    -6
      tests/ut/cpp/dataset/c_api_dataset_coco_test.cc
  10. +6
    -6
      tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc
  11. +4
    -4
      tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc
  12. +17
    -17
      tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc
  13. +61
    -61
      tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
  14. +4
    -4
      tests/ut/cpp/dataset/c_api_dataset_save.cc
  15. +9
    -9
      tests/ut/cpp/dataset/c_api_dataset_voc_test.cc
  16. +8
    -8
      tests/ut/cpp/dataset/c_api_datasets_test.cc
  17. +4
    -4
      tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc
  18. +168
    -51
      tests/ut/cpp/dataset/c_api_samplers_test.cc
  19. +22
    -20
      tests/ut/cpp/dataset/c_api_transforms_test.cc
  20. +20
    -20
      tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc
  21. +1
    -1
      tests/ut/cpp/dataset/c_api_vision_bounding_box_augment_test.cc
  22. +4
    -4
      tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc
  23. +1
    -1
      tests/ut/cpp/dataset/c_api_vision_random_subselect_policy_test.cc
  24. +36
    -36
      tests/ut/cpp/dataset/c_api_vision_random_test.cc
  25. +4
    -4
      tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc
  26. +1
    -1
      tests/ut/cpp/dataset/c_api_vision_uniform_aug_test.cc
  27. +2
    -2
      tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc
  28. +4
    -4
      tests/ut/cpp/dataset/ir_tree_adapter_test.cc
  29. +38
    -38
      tests/ut/cpp/dataset/tree_modifying_function_test.cc

+ 391
- 34
mindspore/ccsrc/minddata/dataset/api/datasets.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -285,7 +285,24 @@ std::shared_ptr<SchemaObj> Schema(const std::string &schema_file) {
// Function to create a AlbumDataset. // Function to create a AlbumDataset.
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, const std::vector<std::string> &column_names, bool decode,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache);

return ds;
}
// Function to create a AlbumDataset.
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache);

return ds;
}
// Function to create a AlbumDataset.
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache); auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache);


@@ -295,7 +312,24 @@ std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::s
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
// Function to create a CelebADataset. // Function to create a CelebADataset.
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler, bool decode,
const std::shared_ptr<Sampler> &sampler, bool decode,
const std::set<std::string> &extensions,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache);

return ds;
}
// Function to create a CelebADataset.
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
bool decode, const std::set<std::string> &extensions,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache);

return ds;
}
// Function to create a CelebADataset.
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler, bool decode,
const std::set<std::string> &extensions, const std::set<std::string> &extensions,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache); auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache);
@@ -305,7 +339,22 @@ std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std:


// Function to create a Cifar10Dataset. // Function to create a Cifar10Dataset.
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache);

return ds;
}
// Function to create a Cifar10Dataset.
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache);

return ds;
}
// Function to create a Cifar10Dataset.
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache); auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache);


@@ -314,7 +363,22 @@ std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const st


// Function to create a Cifar100Dataset. // Function to create a Cifar100Dataset.
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache);

return ds;
}
// Function to create a Cifar100Dataset.
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache);

return ds;
}
// Function to create a Cifar100Dataset.
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache); auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache);


@@ -330,10 +394,26 @@ std::shared_ptr<CLUEDataset> CLUE(const std::vector<std::string> &clue_files, co
return ds; return ds;
} }


// Function to create a CocoDataset.
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache);

return ds;
}
// Function to create a CocoDataset.
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache);

return ds;
}
// Function to create a CocoDataset. // Function to create a CocoDataset.
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode, const std::string &task, const bool &decode,
const std::shared_ptr<SamplerObj> &sampler,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache); auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache);


@@ -353,7 +433,16 @@ std::shared_ptr<CSVDataset> CSV(const std::vector<std::string> &dataset_files, c


// Function to create a ImageFolderDataset. // Function to create a ImageFolderDataset.
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler,
const std::set<std::string> &extensions,
const std::map<std::string, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<ImageFolderDataset>(dataset_dir, decode, sampler, extensions, class_indexing, cache);

return ds;
}
// Function to create a ImageFolderDataset.
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler,
const std::set<std::string> &extensions, const std::set<std::string> &extensions,
const std::map<std::string, int32_t> &class_indexing, const std::map<std::string, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
@@ -361,10 +450,37 @@ std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir,


return ds; return ds;
} }
// Function to create a ImageFolderDataset.
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::set<std::string> &extensions,
const std::map<std::string, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<ImageFolderDataset>(dataset_dir, decode, sampler, extensions, class_indexing, cache);

return ds;
}

// Function to create a ManifestDataset.
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
const std::shared_ptr<Sampler> &sampler,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache);

return ds;
}
// Function to create a ManifestDataset.
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache);


return ds;
}
// Function to create a ManifestDataset. // Function to create a ManifestDataset.
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::string, int32_t> &class_indexing, bool decode, const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache); auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache);
@@ -374,7 +490,22 @@ std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const


// Function to create a MindDataDataset. // Function to create a MindDataDataset.
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
int64_t num_padded) {
auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded);

return ds;
}
// Function to create a MindDataDataset.
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) {
auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded);

return ds;
}
// Function to create a MindDataDataset.
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
int64_t num_padded) { int64_t num_padded) {
auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded); auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded);


@@ -384,7 +515,24 @@ std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const
// Function to create a MindDataDataset. // Function to create a MindDataDataset.
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, const std::vector<std::string> &columns_list,
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
int64_t num_padded) {
auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded);

return ds;
}
// Function to create a MindDataDataset.
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, Sampler *sampler,
nlohmann::json padded_sample, int64_t num_padded) {
auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded);

return ds;
}
// Function to create a MindDataDataset.
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
int64_t num_padded) { int64_t num_padded) {
auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded); auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded);


@@ -393,7 +541,22 @@ std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &datase
#endif #endif
// Function to create a MnistDataset. // Function to create a MnistDataset.
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache);

return ds;
}
// Function to create a MnistDataset.
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache);

return ds;
}
// Function to create a MnistDataset.
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache); auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache);


@@ -419,7 +582,23 @@ std::shared_ptr<TextFileDataset> TextFile(const std::vector<std::string> &datase
// Function to create a VOCDataset. // Function to create a VOCDataset.
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode, const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);

return ds;
}
// Function to create a VOCDataset.
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);

return ds;
}
// Function to create a VOCDataset.
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache); auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);


@@ -831,41 +1010,117 @@ std::shared_ptr<DatasetCache> CreateDatasetCache(session_id_type id, uint64_t me


AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, const std::vector<std::string> &column_names, bool decode,
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler, cache);
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}

AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }


#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler, bool decode,
const std::shared_ptr<Sampler> &sampler, bool decode,
const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, bool decode,
const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler, bool decode,
const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) { const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler, decode, extensions, cache);
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }

Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler, cache);
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }
Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}

Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler, cache);
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }
Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}

CLUEDataset::CLUEDataset(const std::vector<std::string> &dataset_files, const std::string &task, CLUEDataset::CLUEDataset(const std::vector<std::string> &dataset_files, const std::string &task,
const std::string &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, const std::string &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards,
int32_t shard_id, const std::shared_ptr<DatasetCache> &cache) { int32_t shard_id, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CLUENode>(dataset_files, task, usage, num_samples, shuffle, num_shards, shard_id, cache); auto ds = std::make_shared<CLUENode>(dataset_files, task, usage, num_samples, shuffle, num_shards, shard_id, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }

CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
const bool &decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
const bool &decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
const bool &decode, const std::shared_ptr<SamplerObj> &sampler,
const bool &decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler, cache);
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }

CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field_delim, CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field_delim,
const std::vector<std::shared_ptr<CsvBase>> &column_defaults, const std::vector<std::shared_ptr<CsvBase>> &column_defaults,
const std::vector<std::string> &column_names, int64_t num_samples, ShuffleMode shuffle, const std::vector<std::string> &column_names, int64_t num_samples, ShuffleMode shuffle,
@@ -874,8 +1129,21 @@ CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field
num_shards, shard_id, cache); num_shards, shard_id, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }

ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode,
const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<Sampler> &sampler, const std::set<std::string> &extensions,
const std::map<std::string, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) {
// This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false.
bool recursive = false;

// Create logical representation of ImageFolderDataset.
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds =
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler,
const std::set<std::string> &extensions, const std::set<std::string> &extensions,
const std::map<std::string, int32_t> &class_indexing, const std::map<std::string, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
@@ -883,36 +1151,110 @@ ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool deco
bool recursive = false; bool recursive = false;


// Create logical representation of ImageFolderDataset. // Create logical representation of ImageFolderDataset.
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = auto ds =
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler, recursive, extensions, class_indexing, cache);
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }
ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::set<std::string> &extensions,
const std::map<std::string, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) {
// This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false.
bool recursive = false;


// Create logical representation of ImageFolderDataset.
auto sampler_obj = sampler.get().Parse();
auto ds =
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}

ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage,
const std::shared_ptr<Sampler> &sampler,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::string, int32_t> &class_indexing, bool decode, const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache) { const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler, class_indexing, decode, cache);
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}

MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }
MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
int64_t num_padded) { int64_t num_padded) {
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler, padded_sample, num_padded);
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, const std::shared_ptr<Sampler> &sampler,
nlohmann::json padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, Sampler *sampler,
nlohmann::json padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }
MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files, MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, const std::vector<std::string> &columns_list,
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
int64_t num_padded) { int64_t num_padded) {
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler, padded_sample, num_padded);
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }
#endif #endif


MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage,
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler, cache);
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }


@@ -926,8 +1268,23 @@ TextFileDataset::TextFileDataset(const std::vector<std::string> &dataset_files,


VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode, const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds); ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
} }




+ 50
- 60
mindspore/ccsrc/minddata/dataset/api/samplers.cc View File

@@ -20,78 +20,68 @@
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {


/// Function to create a Distributed Sampler.
std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle,
int64_t num_samples, uint32_t seed, int64_t offset,
bool even_dist) {
auto sampler =
std::make_shared<DistributedSamplerObj>(num_shards, shard_id, shuffle, num_samples, seed, offset, even_dist);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// DistributedSampler
DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle, int64_t num_samples,
uint32_t seed, int64_t offset, bool even_dist)
: num_shards_(num_shards),
shard_id_(shard_id),
shuffle_(shuffle),
num_samples_(num_samples),
seed_(seed),
offset_(offset),
even_dist_(even_dist) {}

std::shared_ptr<SamplerObj> DistributedSampler::Parse() {
return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_,
even_dist_);
} }


/// Function to create a PK Sampler.
std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) {
auto sampler = std::make_shared<PKSamplerObj>(num_val, shuffle, num_samples);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// PKSampler
PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples)
: num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {}

std::shared_ptr<SamplerObj> PKSampler::Parse() {
return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_);
} }


/// Function to create a Random Sampler.
std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement, int64_t num_samples) {
auto sampler = std::make_shared<RandomSamplerObj>(replacement, num_samples);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// RandomSampler
RandomSampler::RandomSampler(bool replacement, int64_t num_samples)
: replacement_(replacement), num_samples_(num_samples) {}

std::shared_ptr<SamplerObj> RandomSampler::Parse() {
return std::make_shared<RandomSamplerObj>(replacement_, num_samples_);
} }


/// Function to create a Sequential Sampler.
std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index, int64_t num_samples) {
auto sampler = std::make_shared<SequentialSamplerObj>(start_index, num_samples);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// SequentialSampler
SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples)
: start_index_(start_index), num_samples_(num_samples) {}

std::shared_ptr<SamplerObj> SequentialSampler::Parse() {
return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_);
} }


/// Function to create a Subset Random Sampler.
std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples) {
auto sampler = std::make_shared<SubsetSamplerObj>(std::move(indices), num_samples);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// SubsetSampler
SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples)
: indices_(indices), num_samples_(num_samples) {}

std::shared_ptr<SamplerObj> SubsetSampler::Parse() {
return std::make_shared<SubsetSamplerObj>(indices_, num_samples_);
} }


/// Function to create a Subset Random Sampler.
std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples) {
auto sampler = std::make_shared<SubsetRandomSamplerObj>(std::move(indices), num_samples);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// SubsetRandomSampler
SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples)
: SubsetSampler(indices, num_samples) {}

std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() {
return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_);
} }


/// Function to create a Weighted Random Sampler.
std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples,
bool replacement) {
auto sampler = std::make_shared<WeightedRandomSamplerObj>(std::move(weights), num_samples, replacement);
// Input validation
if (sampler->ValidateParams().IsError()) {
return nullptr;
}
return sampler;
// WeightedRandomSampler
WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement)
: weights_(weights), num_samples_(num_samples), replacement_(replacement) {}

std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() {
return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_);
} }


} // namespace dataset } // namespace dataset


+ 5
- 5
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc View File

@@ -138,7 +138,7 @@ Status ValidateDatasetShardParams(const std::string &dataset_name, int32_t num_s


// Helper function to validate dataset sampler parameter // Helper function to validate dataset sampler parameter
Status ValidateDatasetSampler(const std::string &dataset_name, const std::shared_ptr<SamplerObj> &sampler) { Status ValidateDatasetSampler(const std::string &dataset_name, const std::shared_ptr<SamplerObj> &sampler) {
if (sampler == nullptr) {
if (sampler == nullptr || sampler->ValidateParams().IsError()) {
std::string err_msg = dataset_name + ": Sampler is not constructed correctly, sampler: nullptr"; std::string err_msg = dataset_name + ": Sampler is not constructed correctly, sampler: nullptr";
MS_LOG(ERROR) << err_msg; MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); RETURN_STATUS_SYNTAX_ERROR(err_msg);
@@ -191,17 +191,17 @@ std::shared_ptr<SamplerObj> SelectSampler(int64_t num_samples, bool shuffle, int
if (shuffle) { if (shuffle) {
if (num_shards > 1) { if (num_shards > 1) {
// If shuffle enabled, sharding enabled, use distributed random sampler // If shuffle enabled, sharding enabled, use distributed random sampler
return DistributedSampler(num_shards, shard_id, shuffle, num_samples);
return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse();
} }
// If shuffle enabled, sharding disabled, use random sampler // If shuffle enabled, sharding disabled, use random sampler
return RandomSampler(num_samples >= 0, num_samples);
return RandomSampler(num_samples >= 0, num_samples).Parse();
} }
if (num_shards > 1) { if (num_shards > 1) {
// If shuffle disabled, sharding enabled, use distributed sequential sampler // If shuffle disabled, sharding enabled, use distributed sequential sampler
return DistributedSampler(num_shards, shard_id, shuffle, num_samples);
return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse();
} }
// If shuffle disabled, sharding disabled, use sequential sampler // If shuffle disabled, sharding disabled, use sequential sampler
return SequentialSampler(0, num_samples);
return SequentialSampler(0, num_samples).Parse();
} }


// Constructor to initialize the cache // Constructor to initialize the cache


+ 398
- 37
mindspore/ccsrc/minddata/dataset/include/datasets.h View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -585,7 +585,13 @@ class AlbumDataset : public Dataset {
public: public:
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names = {}, bool decode = false, const std::vector<std::string> &column_names = {}, bool decode = false,
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~AlbumDataset() = default; ~AlbumDataset() = default;
}; };
@@ -597,20 +603,53 @@ class AlbumDataset : public Dataset {
/// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns. /// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns.
/// (default = {}) /// (default = {})
/// \param[in] decode the option to decode the images in dataset (default = false) /// \param[in] decode the option to decode the images in dataset (default = false)
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names = {}, bool decode = false, const std::vector<std::string> &column_names = {}, bool decode = false,
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create an AlbumDataset
/// \notes The generated dataset is specified through setting a schema
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] data_schema Path to dataset schema file
/// \param[in] column_names Column names used to specify columns to load
/// \param[in] decode the option to decode the images in dataset
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create an AlbumDataset
/// \notes The generated dataset is specified through setting a schema
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] data_schema Path to dataset schema file
/// \param[in] column_names Column names used to specify columns to load
/// \param[in] decode the option to decode the images in dataset
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);


#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
class CelebADataset : public Dataset { class CelebADataset : public Dataset {
public: public:
explicit CelebADataset(const std::string &dataset_dir, const std::string &usage = "all", explicit CelebADataset(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), bool decode = false,
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
bool decode = false, const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
bool decode = false, const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit CelebADataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler, bool decode = false,
const std::set<std::string> &extensions = {}, const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~CelebADataset() = default; ~CelebADataset() = default;
@@ -621,21 +660,54 @@ class CelebADataset : public Dataset {
/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. /// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type.
/// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage One of "all", "train", "valid" or "test" (default = "all"). /// \param[in] usage One of "all", "train", "valid" or "test" (default = "all").
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] decode Decode the images after reading (default=false). /// \param[in] decode Decode the images after reading (default=false).
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}). /// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage = "all", std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), bool decode = false,
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
bool decode = false, const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a CelebADataset
/// \notes The generated dataset has two columns ['image', 'attr'].
/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage One of "all", "train", "valid" or "test"
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
bool decode = false, const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a CelebADataset
/// \notes The generated dataset has two columns ['image', 'attr'].
/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage One of "all", "train", "valid" or "test"
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler, bool decode = false,
const std::set<std::string> &extensions = {}, const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);


class Cifar10Dataset : public Dataset { class Cifar10Dataset : public Dataset {
public: public:
explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage = "all", explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~Cifar10Dataset() = default; ~Cifar10Dataset() = default;
}; };
@@ -644,18 +716,43 @@ class Cifar10Dataset : public Dataset {
/// \notes The generated dataset has two columns ["image", "label"] /// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of CIFAR10, can be "train", "test" or "all" (default = "all"). /// \param[in] usage of CIFAR10, can be "train", "test" or "all" (default = "all").
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage = "all", std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a Cifar10 Dataset
/// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of CIFAR10, can be "train", "test" or "all"
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a Cifar10 Dataset
/// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of CIFAR10, can be "train", "test" or "all"
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);


class Cifar100Dataset : public Dataset { class Cifar100Dataset : public Dataset {
public: public:
explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage = "all", explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~Cifar100Dataset() = default; ~Cifar100Dataset() = default;
}; };
@@ -664,12 +761,32 @@ class Cifar100Dataset : public Dataset {
/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] /// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of CIFAR100, can be "train", "test" or "all" (default = "all"). /// \param[in] usage of CIFAR100, can be "train", "test" or "all" (default = "all").
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage = "all", std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a Cifar100 Dataset
/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of CIFAR100, can be "train", "test" or "all".
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a Cifar100 Dataset
/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of CIFAR100, can be "train", "test" or "all".
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);


class CLUEDataset : public Dataset { class CLUEDataset : public Dataset {
@@ -707,7 +824,12 @@ std::shared_ptr<CLUEDataset> CLUE(const std::vector<std::string> &dataset_files,
class CocoDataset : public Dataset { class CocoDataset : public Dataset {
public: public:
CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection", CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection",
const bool &decode = false, const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const bool &decode = false, const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
const bool &decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache = nullptr);
CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
const bool &decode, const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~CocoDataset() = default; ~CocoDataset() = default;
}; };
@@ -725,13 +847,53 @@ class CocoDataset : public Dataset {
/// \param[in] annotation_file Path to the annotation json /// \param[in] annotation_file Path to the annotation json
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'
/// \param[in] decode Decode the images after reading /// \param[in] decode Decode the images after reading
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task = "Detection", const bool &decode = false, const std::string &task = "Detection", const bool &decode = false,
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a CocoDataset
/// \notes The generated dataset has multi-columns :
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
/// ['iscrowd', dtype=uint32]].
/// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]].
/// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32],
/// ['num_keypoints', dtype=uint32]].
/// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
/// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]].
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] annotation_file Path to the annotation json
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'
/// \param[in] decode Decode the images after reading
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a CocoDataset
/// \notes The generated dataset has multi-columns :
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
/// ['iscrowd', dtype=uint32]].
/// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]].
/// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32],
/// ['num_keypoints', dtype=uint32]].
/// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
/// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]].
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] annotation_file Path to the annotation json
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'
/// \param[in] decode Decode the images after reading
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);


class CSVDataset : public Dataset { class CSVDataset : public Dataset {
@@ -774,7 +936,16 @@ std::shared_ptr<CSVDataset> CSV(const std::vector<std::string> &dataset_files, c
class ImageFolderDataset : public Dataset { class ImageFolderDataset : public Dataset {
public: public:
explicit ImageFolderDataset(const std::string &dataset_dir, bool decode = false, explicit ImageFolderDataset(const std::string &dataset_dir, bool decode = false,
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler,
const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit ImageFolderDataset(const std::string &dataset_dir, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::set<std::string> &extensions = {}, const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {}, const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
@@ -787,14 +958,46 @@ class ImageFolderDataset : public Dataset {
/// The generated dataset has two columns ["image", "label"] /// The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] decode A flag to decode in ImageFolder /// \param[in] decode A flag to decode in ImageFolder
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] extensions File extensions to be read /// \param[in] extensions File extensions to be read
/// \param[in] class_indexing a class name to label map /// \param[in] class_indexing a class name to label map
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ImageFolderDataset /// \return Shared pointer to the current ImageFolderDataset
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode = false,
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
std::shared_ptr<ImageFolderDataset> ImageFolder(
const std::string &dataset_dir, bool decode = false,
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::set<std::string> &extensions = {}, const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create an ImageFolderDataset
/// \notes A source dataset that reads images from a tree of directories
/// All images within one folder have the same label
/// The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] decode A flag to decode in ImageFolder
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] extensions File extensions to be read
/// \param[in] class_indexing a class name to label map
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ImageFolderDataset
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler,
const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create an ImageFolderDataset
/// \notes A source dataset that reads images from a tree of directories
/// All images within one folder have the same label
/// The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] decode A flag to decode in ImageFolder
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] extensions File extensions to be read
/// \param[in] class_indexing a class name to label map
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ImageFolderDataset
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::set<std::string> &extensions = {}, const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {}, const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
@@ -802,7 +1005,14 @@ std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir,
class ManifestDataset : public Dataset { class ManifestDataset : public Dataset {
public: public:
explicit ManifestDataset(const std::string &dataset_file, const std::string &usage = "train", explicit ManifestDataset(const std::string &dataset_file, const std::string &usage = "train",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit ManifestDataset(const std::string &dataset_file, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~ManifestDataset() = default; ~ManifestDataset() = default;
@@ -812,7 +1022,8 @@ class ManifestDataset : public Dataset {
/// \notes The generated dataset has two columns ["image", "label"] /// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_file The dataset file to be read /// \param[in] dataset_file The dataset file to be read
/// \param[in] usage Need "train", "eval" or "inference" data (default="train") /// \param[in] usage Need "train", "eval" or "inference" data (default="train")
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder /// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder
/// names will be sorted alphabetically and each class will be given a unique index starting from 0). /// names will be sorted alphabetically and each class will be given a unique index starting from 0).
@@ -820,19 +1031,56 @@ class ManifestDataset : public Dataset {
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ManifestDataset /// \return Shared pointer to the current ManifestDataset
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage = "train", std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage = "train",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::map<std::string, int32_t> &class_indexing = {},
bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a ManifestDataset
/// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_file The dataset file to be read
/// \param[in] usage Need "train", "eval" or "inference" data
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ManifestDataset
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
const std::map<std::string, int32_t> &class_indexing = {},
bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a ManifestDataset
/// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_file The dataset file to be read
/// \param[in] usage Need "train", "eval" or "inference" data
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
/// \param[in] decode Decode the images after reading (default=false).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ManifestDataset
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::string, int32_t> &class_indexing = {}, const std::map<std::string, int32_t> &class_indexing = {},
bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr); bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr);


class MindDataDataset : public Dataset { class MindDataDataset : public Dataset {
public: public:
explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list = {}, explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list = {},
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample = nullptr,
int64_t num_padded = 0);
explicit MindDataDataset(const std::vector<std::string> &dataset_files, explicit MindDataDataset(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list = {}, const std::vector<std::string> &columns_list = {},
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
explicit MindDataDataset(const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list,
Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
explicit MindDataDataset(const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample = nullptr,
int64_t num_padded = 0);
~MindDataDataset() = default; ~MindDataDataset() = default;
}; };


@@ -840,7 +1088,8 @@ class MindDataDataset : public Dataset {
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
/// in the same path will be found and loaded automatically. /// in the same path will be found and loaded automatically.
/// \param[in] columns_list List of columns to be read (default={}) /// \param[in] columns_list List of columns to be read (default={})
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()),
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
@@ -848,13 +1097,38 @@ class MindDataDataset : public Dataset {
/// \return Shared pointer to the current MindDataDataset /// \return Shared pointer to the current MindDataDataset
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
const std::vector<std::string> &columns_list = {}, const std::vector<std::string> &columns_list = {},
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
/// \brief Function to create a MindDataDataset
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
/// in the same path will be found and loaded automatically.
/// \param[in] columns_list List of columns to be read
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
Sampler *sampler, nlohmann::json padded_sample = nullptr,
int64_t num_padded = 0);
/// \brief Function to create a MindDataDataset
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
/// in the same path will be found and loaded automatically.
/// \param[in] columns_list List of columns to be read
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);


/// \brief Function to create a MindDataDataset /// \brief Function to create a MindDataDataset
/// \param[in] dataset_files List of dataset files to be read directly. /// \param[in] dataset_files List of dataset files to be read directly.
/// \param[in] columns_list List of columns to be read (default={}) /// \param[in] columns_list List of columns to be read (default={})
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()),
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
@@ -862,14 +1136,42 @@ std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
/// \return Shared pointer to the current MindDataDataset /// \return Shared pointer to the current MindDataDataset
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list = {}, const std::vector<std::string> &columns_list = {},
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
/// \brief Function to create a MindDataDataset
/// \param[in] dataset_files List of dataset files to be read directly.
/// \param[in] columns_list List of columns to be read
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, Sampler *sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
/// \brief Function to create a MindDataDataset
/// \param[in] dataset_files List of dataset files to be read directly.
/// \param[in] columns_list List of columns to be read
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
#endif #endif


class MnistDataset : public Dataset { class MnistDataset : public Dataset {
public: public:
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all", explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~MnistDataset() = default; ~MnistDataset() = default;
}; };
@@ -878,12 +1180,32 @@ class MnistDataset : public Dataset {
/// \notes The generated dataset has two columns ["image", "label"] /// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all"). /// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all").
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset /// \return Shared pointer to the current MnistDataset
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all", std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a MnistDataset
/// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of MNIST, can be "train", "test" or "all"
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a MnistDataset
/// \notes The generated dataset has two columns ["image", "label"]
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] usage of MNIST, can be "train", "test" or "all"
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
/// \brief Function to create a ConcatDataset /// \brief Function to create a ConcatDataset
@@ -909,8 +1231,6 @@ class RandomDataDataset : public Dataset {
/// \param[in] total_rows Number of rows for the dataset to generate (default=0, number of rows is random) /// \param[in] total_rows Number of rows for the dataset to generate (default=0, number of rows is random)
/// \param[in] schema SchemaObj to set column type, data type and data shape /// \param[in] schema SchemaObj to set column type, data type and data shape
/// \param[in] columns_list List of columns to be read (default={}, read all columns) /// \param[in] columns_list List of columns to be read (default={}, read all columns)
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
template <typename T = std::shared_ptr<SchemaObj>> template <typename T = std::shared_ptr<SchemaObj>>
@@ -1027,7 +1347,14 @@ class VOCDataset : public Dataset {
public: public:
explicit VOCDataset(const std::string &dataset_dir, const std::string &task = "Segmentation", explicit VOCDataset(const std::string &dataset_dir, const std::string &task = "Segmentation",
const std::string &usage = "train", const std::map<std::string, int32_t> &class_indexing = {}, const std::string &usage = "train", const std::map<std::string, int32_t> &class_indexing = {},
bool decode = false, const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
bool decode = false, const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);
~VOCDataset() = default; ~VOCDataset() = default;
}; };
@@ -1042,14 +1369,48 @@ class VOCDataset : public Dataset {
/// \param[in] usage The type of data list text file to be read (default = "train"). /// \param[in] usage The type of data list text file to be read (default = "train").
/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task /// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task
/// \param[in] decode Decode the images after reading /// \param[in] decode Decode the images after reading
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given,
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset /// \return Shared pointer to the current Dataset
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task = "Segmentation", std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task = "Segmentation",
const std::string &usage = "train", const std::string &usage = "train",
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a VOCDataset
/// \notes The generated dataset has multi-columns :
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32],
/// ['difficult', dtype=uint32], ['truncate', dtype=uint32]].
/// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]].
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection"
/// \param[in] usage The type of data list text file to be read.
/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task
/// \param[in] decode Decode the images after reading
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr);
/// \brief Function to create a VOCDataset
/// \notes The generated dataset has multi-columns :
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32],
/// ['difficult', dtype=uint32], ['truncate', dtype=uint32]].
/// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]].
/// \param[in] dataset_dir Path to the root directory that contains the dataset
/// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection"
/// \param[in] usage The type of data list text file to be read.
/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task
/// \param[in] decode Decode the images after reading
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
const std::map<std::string, int32_t> &class_indexing, bool decode,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr); const std::shared_ptr<DatasetCache> &cache = nullptr);


/// \brief Function the create a cache to be attached to a dataset /// \brief Function the create a cache to be attached to a dataset


+ 206
- 57
mindspore/ccsrc/minddata/dataset/include/samplers.h View File

@@ -26,76 +26,225 @@
namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {


class DistributedSamplerObj;
class PKSamplerObj;
class PreBuiltSamplerObj;
class RandomSamplerObj;
class SequentialSamplerObj;
class SubsetSamplerObj;
class SubsetRandomSamplerObj;
class WeightedRandomSamplerObj;

/// Function to create a Distributed Sampler.
/// \notes A Sampler that access a shard of the dataset.
/// \param[in] num_shards - Number of shards to divide the dataset into.
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
/// \param[in] shuffle - If true, the indices are shuffled.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \param[in] seed - The seed in use when shuffle is true.
/// \param[in] offset - The starting position where access to elements in the dataset begins.
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
/// If false the total rows returned by all the shards would not have overlap.
/// \return Shared pointer to the current Sampler.
std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true,
int64_t num_samples = 0, uint32_t seed = 1,
int64_t offset = -1, bool even_dist = true);

/// Function to create a PK Sampler.
// Abstract class to represent a sampler in the data pipeline.
/// \class Sampler samplers.h
/// \brief An abstract base class to represent a sampler in the data pipeline.
class Sampler : std::enable_shared_from_this<Sampler> {
friend class AlbumDataset;
friend class CelebADataset;
friend class Cifar10Dataset;
friend class Cifar100Dataset;
friend class CLUEDataset;
friend class CocoDataset;
friend class CSVDataset;
friend class ImageFolderDataset;
friend class ManifestDataset;
friend class MindDataDataset;
friend class MnistDataset;
friend class RandomDataDataset;
friend class TextFileDataset;
friend class TFRecordDataset;
friend class VOCDataset;
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
Sampler() {}

/// \brief Destructor
~Sampler() = default;

/// \brief A virtual function to add a child sampler.
/// \param[in] child The child sampler to be added as a children of this sampler.
virtual void AddChild(std::shared_ptr<Sampler> child) { children_.push_back(child); }

protected:
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<SamplerObj> Parse() = 0;

std::vector<std::shared_ptr<Sampler>> children_;
};

/// \brief A class to represent a Distributed Sampler in the data pipeline.
/// \notes A Sampler that accesses a shard of the dataset.
class DistributedSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] num_shards - Number of shards to divide the dataset into.
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
/// \param[in] shuffle - If true, the indices are shuffled.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \param[in] seed - The seed in use when shuffle is true.
/// \param[in] offset - The starting position where access to elements in the dataset begins.
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
/// If false the total rows returned by all the shards would not have overlap.
explicit DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, int64_t num_samples = 0,
uint32_t seed = 1, int64_t offset = -1, bool even_dist = true);
/// \brief Destructor.
~DistributedSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;

private:
int64_t num_shards_;
int64_t shard_id_;
bool shuffle_;
int64_t num_samples_;
uint32_t seed_;
int64_t offset_;
bool even_dist_;
};

/// \brief A class to represent a PK Sampler in the data pipeline.
/// \notes Samples K elements for each P class in the dataset. /// \notes Samples K elements for each P class in the dataset.
/// This will sample all classes. /// This will sample all classes.
/// \param[in] num_val - Number of elements to sample for each class.
/// \param[in] shuffle - If true, the class IDs are shuffled.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \return Shared pointer to the current Sampler.
std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);
class PKSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] num_val - Number of elements to sample for each class.
/// \param[in] shuffle - If true, the class IDs are shuffled.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);

/// \brief Destructor.
~PKSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;


/// Function to create a Random Sampler.
private:
int64_t num_val_;
bool shuffle_;
int64_t num_samples_;
};

/// \brief A class to represent a Random Sampler in the data pipeline.
/// \notes Samples the elements randomly. /// \notes Samples the elements randomly.
/// \param[in] replacement - If true, put the sample ID back for the next draw.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \return Shared pointer to the current Sampler.
std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement = false, int64_t num_samples = 0);
class RandomSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] replacement - If true, put the sample ID back for the next draw.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit RandomSampler(bool replacement = false, int64_t num_samples = 0);

/// \brief Destructor.
~RandomSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;


/// Function to create a Sequential Sampler.
private:
bool replacement_;
int64_t num_samples_;
};

/// \brief A class to represent a Sequential Sampler in the data pipeline.
/// \notes Samples the dataset elements sequentially, same as not having a sampler. /// \notes Samples the dataset elements sequentially, same as not having a sampler.
/// \param[in] start_index - Index to start sampling at (default to start at first id).
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \return Shared pointer to the current Sampler.
std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);
class SequentialSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] start_index - Index to start sampling at (default to start at first id).
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);

/// \brief Destructor.
~SequentialSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;


/// Function to create a Subset Sampler.
private:
int64_t start_index_;
int64_t num_samples_;
};

/// \brief A class to represent a Subset Sampler in the data pipeline.
/// \notes Samples the elements from a sequence of indices. /// \notes Samples the elements from a sequence of indices.
/// \param[in] indices - A vector sequence of indices.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \return Shared pointer to the current Sampler.
std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
class SubsetSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] indices - A vector sequence of indices.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);

/// \brief Destructor.
~SubsetSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;


/// Function to create a Subset Random Sampler.
std::vector<int64_t> indices_;
int64_t num_samples_;
};

/// \brief A class to represent a Subset Random Sampler in the data pipeline.
/// \notes Samples the elements randomly from a sequence of indices. /// \notes Samples the elements randomly from a sequence of indices.
/// \param[in] indices - A vector sequence of indices.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \return Shared pointer to the current Sampler.
std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
class SubsetRandomSampler : public SubsetSampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] indices - A vector sequence of indices.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
explicit SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);

/// \brief Destructor.
~SubsetRandomSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
};


/// Function to create a Weighted Random Sampler.
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
/// \notes Samples the elements from [0, len(weights) - 1] randomly with the given /// \notes Samples the elements from [0, len(weights) - 1] randomly with the given
/// weights (probabilities). /// weights (probabilities).
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \param[in] replacement - If true, put the sample ID back for the next draw.
/// \return Shared pointer to the current Sampler.
std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0,
bool replacement = true);
class WeightedRandomSampler : public Sampler {
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);

public:
/// \brief Constructor
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
/// \param[in] num_samples - The number of samples to draw (default to all elements).
/// \param[in] replacement - If true, put the sample ID back for the next draw.
explicit WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0, bool replacement = true);

/// \brief Destructor.
~WeightedRandomSampler() = default;

protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;

private:
std::vector<double> weights_;
int64_t num_samples_;
bool replacement_;
};


} // namespace dataset } // namespace dataset
} // namespace mindspore } // namespace mindspore


+ 16
- 16
tests/ut/cpp/dataset/c_api_cache_test.cc View File

@@ -57,7 +57,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCApiNestedCache) {


// Create an ImageFolder Dataset, this folder_path only has 2 images in it // Create an ImageFolder Dataset, this folder_path only has 2 images in it
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -85,7 +85,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheImageFolderCApi) {


// Create an ImageFolder Dataset, this folder_path only has 2 images in it // Create an ImageFolder Dataset, this folder_path only has 2 images in it
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -128,7 +128,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCocoCApi) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train/"; std::string folder_path = datasets_root_path_ + "/testCOCO/train/";
std::string annotation_file_path = datasets_root_path_ + "/testCOCO/annotations/train.json"; std::string annotation_file_path = datasets_root_path_ + "/testCOCO/annotations/train.json";
std::shared_ptr<Dataset> ds = std::shared_ptr<Dataset> ds =
Coco(folder_path, annotation_file_path, "Detection", false, RandomSampler(), some_cache);
Coco(folder_path, annotation_file_path, "Detection", false, std::make_shared<RandomSampler>(), some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -169,7 +169,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheMnistCApi) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10), some_cache);
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -210,7 +210,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) {


// Create a CelebA Dataset, this folder_path has 4 records in it // Create a CelebA Dataset, this folder_path has 4 records in it
std::string folder_path = datasets_root_path_ + "/testCelebAData/"; std::string folder_path = datasets_root_path_ + "/testCelebAData/";
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", RandomSampler(false, 10), false, {}, some_cache);
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", std::make_shared<RandomSampler>(false, 10), false, {}, some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -251,7 +251,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheManifestCApi) {


// Create a Manifest Dataset, this file_path has 2 records in it // Create a Manifest Dataset, this file_path has 2 records in it
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), {}, false, some_cache);
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), {}, false, some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -292,7 +292,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar10CApi) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10), some_cache);
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -333,7 +333,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar100CApi) {


// Create a Cifar100 Dataset // Create a Cifar100 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10), some_cache);
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -374,7 +374,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) {


// Create a VOC Dataset, this folder_path has 9 records in it // Create a VOC Dataset, this folder_path has 9 records in it
std::string folder_path = datasets_root_path_ + "/testVOC2012/"; std::string folder_path = datasets_root_path_ + "/testVOC2012/";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, RandomSampler(), some_cache);
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<RandomSampler>(), some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -417,7 +417,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheAlbumCApi) {
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"}; std::vector<std::string> column_names = {"image", "label", "id"};
// Create a Album Dataset, 7 records in it // Create a Album Dataset, 7 records in it
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, false, RandomSampler(), some_cache);
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, false, std::make_shared<RandomSampler>(), some_cache);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -770,9 +770,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) {


// Create an ImageFolder Dataset, this folder_path only has 2 images in it // Create an ImageFolder Dataset, this folder_path only has 2 images in it
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create and launch the Execution Tree for ds1 // Create and launch the Execution Tree for ds1
@@ -824,9 +824,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) {
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
// The first pipeline is ImageFolder with RandomSampler, the second pipeline is ImageFolder with SequentialSampler // The first pipeline is ImageFolder with RandomSampler, the second pipeline is ImageFolder with SequentialSampler
// Since sampler does not influence the data in the source, these two pipelines can share a common cache. // Since sampler does not influence the data in the source, these two pipelines can share a common cache.
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, SequentialSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(), {}, {}, some_cache);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create and launch the Execution Tree for ds1 // Create and launch the Execution Tree for ds1
@@ -874,9 +874,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) {


// Create an ImageFolder Dataset, this folder_path only has 2 images in it // Create an ImageFolder Dataset, this folder_path only has 2 images in it
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create and launch the Execution Tree for ds1 // Create and launch the Execution Tree for ds1


+ 3
- 3
tests/ut/cpp/dataset/c_api_dataset_album_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"}; std::vector<std::string> column_names = {"image", "label", "id"};
// Create a Album Dataset // Create a Album Dataset
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1));
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -206,7 +206,7 @@ TEST_F(MindDataTestPipeline, TestAlbumError) {
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"}; std::vector<std::string> column_names = {"image", "label", "id"};
// Create an Album Dataset // Create an Album Dataset
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1));
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset


+ 9
- 9
tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -64,8 +64,8 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) {


// Create two Cifar10 Dataset // Create two Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds1 = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds1 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


@@ -168,7 +168,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Dataset) {


// Create a Cifar100 Dataset // Create a Cifar100 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -203,7 +203,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Getters) {


// Create a Cifar100 Dataset // Create a Cifar100 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


std::vector<std::string> column_names = {"image", "coarse_label", "fine_label"}; std::vector<std::string> column_names = {"image", "coarse_label", "fine_label"};
@@ -230,7 +230,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100DatasetFail."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100DatasetFail.";


// Create a Cifar100 Dataset // Create a Cifar100 Dataset
std::shared_ptr<Dataset> ds = Cifar100("", "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar100("", "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -243,7 +243,7 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail.";


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::shared_ptr<Dataset> ds = Cifar10("", "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10("", "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetWithWrongSamplerFail) {


// Create a Cifar100 Dataset // Create a Cifar100 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, -10));
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, -10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset


+ 6
- 6
tests/ut/cpp/dataset/c_api_dataset_coco_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -138,7 +138,7 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";


std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -220,7 +220,7 @@ TEST_F(MindDataTestPipeline, TestCocoKeypoint) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json";


std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, SequentialSampler(0, 2));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -275,7 +275,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json";


std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -333,7 +333,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanopticGetClassIndex) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json";


std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
@@ -355,7 +355,7 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";


std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset


+ 6
- 6
tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc View File

@@ -1,6 +1,6 @@


/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -29,7 +29,7 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn.";
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Rename operation on ds // Create a Rename operation on ds
@@ -65,7 +65,7 @@ TEST_F(MindDataTestPipeline, TestIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 4));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestIteratorReOrder) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder.";
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(false, 4));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(false, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Take operation on ds // Create a Take operation on ds
@@ -144,7 +144,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns.";
// Create a VOC Dataset // Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -187,7 +187,7 @@ TEST_F(MindDataTestPipeline, TestIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 4));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Pass wrong column name // Pass wrong column name


+ 4
- 4
tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -147,7 +147,7 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) {


std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset // Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), {}, true);
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), {}, true);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -218,7 +218,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
std::vector<int64_t> expected_label = {111, 222}; std::vector<int64_t> expected_label = {111, 222};


// Create a Manifest Dataset // Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), map, true);
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), map, true);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
@@ -261,7 +261,7 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {


std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset // Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", SequentialSampler(0, 1), {}, true);
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<SequentialSampler>(0, 1), {}, true);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset


+ 17
- 17
tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -182,7 +182,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess5) {
// Pass one mindrecord shard file to parse dataset info, and search for other mindrecord files with same dataset info, // Pass one mindrecord shard file to parse dataset info, and search for other mindrecord files with same dataset info,
// thus all records in imagenet.mindrecord0 ~ imagenet.mindrecord3 will be read // thus all records in imagenet.mindrecord0 ~ imagenet.mindrecord3 will be read
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds = MindData(file_path1, {}, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = MindData(file_path1, {}, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -223,26 +223,26 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) {
std::vector<std::string> file_list = {file_path1}; std::vector<std::string> file_list = {file_path1};


// Check sequential sampler, output number is 5 // Check sequential sampler, output number is 5
std::shared_ptr<Dataset> ds1 = MindData(file_list, {}, SequentialSampler(0, 10));
std::shared_ptr<Dataset> ds1 = MindData(file_list, {}, std::make_shared<SequentialSampler>(0, 10));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Check random sampler, output number is 5, same rows with file // Check random sampler, output number is 5, same rows with file
std::shared_ptr<Dataset> ds2 = MindData(file_list, {}, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds2 = MindData(file_list, {}, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Check pk sampler, output number is 2, get 2 samples with label 0 // Check pk sampler, output number is 2, get 2 samples with label 0
std::shared_ptr<Dataset> ds3 = MindData(file_list, {}, PKSampler(2, false, 10));
std::shared_ptr<Dataset> ds3 = MindData(file_list, {}, std::make_shared<PKSampler>(2, false, 10));
EXPECT_NE(ds3, nullptr); EXPECT_NE(ds3, nullptr);


// Check distributed sampler, output number is 3, get 3 samples in shard 0 // Check distributed sampler, output number is 3, get 3 samples in shard 0
std::shared_ptr<Dataset> ds4 = MindData(file_list, {}, DistributedSampler(2, 0, false, 10));
std::shared_ptr<Dataset> ds4 = MindData(file_list, {}, std::make_shared<DistributedSampler>(2, 0, false, 10));
EXPECT_NE(ds4, nullptr); EXPECT_NE(ds4, nullptr);


// Check distributed sampler get 3 samples with indice 0, 1 ,2 // Check distributed sampler get 3 samples with indice 0, 1 ,2
std::shared_ptr<Dataset> ds5 = MindData(file_list, {}, SubsetRandomSampler({0, 1, 2}, 10));
std::shared_ptr<Dataset> ds5 = MindData(file_list, {}, new SubsetRandomSampler({0, 1, 2}, 10));
EXPECT_NE(ds5, nullptr); EXPECT_NE(ds5, nullptr);


std::shared_ptr<Dataset> ds6 = MindData(file_list, {}, SubsetSampler({1, 2}, 10));
std::shared_ptr<Dataset> ds6 = MindData(file_list, {}, new SubsetSampler({1, 2}, 10));
EXPECT_NE(ds5, nullptr); EXPECT_NE(ds5, nullptr);


std::vector<std::shared_ptr<Dataset>> ds = {ds1, ds2, ds3, ds4, ds5, ds6}; std::vector<std::shared_ptr<Dataset>> ds = {ds1, ds2, ds3, ds4, ds5, ds6};
@@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) {
// Pass a list of mindrecord file name, files in list will be read directly but not search for related files // Pass a list of mindrecord file name, files in list will be read directly but not search for related files
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::vector<std::string> file_list = {file_path1}; std::vector<std::string> file_list = {file_path1};
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Skip operation on ds, skip original data in mindrecord and get padded samples // Create a Skip operation on ds, skip original data in mindrecord and get padded samples
@@ -332,7 +332,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
// Pass a list of mindrecord file name, files in list will be read directly but not search for related files // Pass a list of mindrecord file name, files in list will be read directly but not search for related files
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::vector<std::string> file_list = {file_path1}; std::vector<std::string> file_list = {file_path1};
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


std::vector<DataType> types = ds->GetOutputTypes(); std::vector<DataType> types = ds->GetOutputTypes();
@@ -400,12 +400,12 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) {
// Pass a list of mindrecord file name, files in list will be read directly but not search for related files // Pass a list of mindrecord file name, files in list will be read directly but not search for related files
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::vector<std::string> file_list = {file_path1}; std::vector<std::string> file_list = {file_path1};
std::shared_ptr<Dataset> ds1 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
std::shared_ptr<Dataset> ds1 = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);
ds1 = ds1->Skip(5); ds1 = ds1->Skip(5);
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


std::shared_ptr<Dataset> ds2 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
std::shared_ptr<Dataset> ds2 = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);
ds2 = ds2->Skip(5); ds2 = ds2->Skip(5);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);
@@ -534,7 +534,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail3) {


// Create a MindData Dataset with unsupported sampler // Create a MindData Dataset with unsupported sampler
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, WeightedRandomSampler({1, 1, 1, 1}));
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, new WeightedRandomSampler({1, 1, 1, 1}));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -558,7 +558,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {


// Create a MindData Dataset // Create a MindData Dataset
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, RandomSampler(), nullptr, 2);
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, std::make_shared<RandomSampler>(), nullptr, 2);
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {


// Create a MindData Dataset // Create a MindData Dataset
std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, RandomSampler(), pad, -2);
std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, std::make_shared<RandomSampler>(), pad, -2);
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -583,7 +583,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {


// Create a MindData Dataset // Create a MindData Dataset
std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, RandomSampler(), pad, 1);
std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, std::make_shared<RandomSampler>(), pad, 1);
EXPECT_NE(ds3, nullptr); EXPECT_NE(ds3, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -598,7 +598,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {


// Create a MindData Dataset // Create a MindData Dataset
std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds4 = MindData(file_path4, {"file_name", "label"}, RandomSampler(), pad2, 1);
std::shared_ptr<Dataset> ds4 = MindData(file_path4, {"file_name", "label"}, std::make_shared<RandomSampler>(), pad2, 1);
EXPECT_NE(ds4, nullptr); EXPECT_NE(ds4, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset


+ 61
- 61
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc View File

@@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -118,7 +118,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess1) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -154,7 +154,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -192,7 +192,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail1) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -212,7 +212,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail2) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -232,7 +232,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail3) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -252,7 +252,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail4) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -272,7 +272,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail5) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -292,7 +292,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail6) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);
// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, -2, 3}); ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, -2, 3});
@@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail7) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a BucketBatchByLength operation on ds // Create a BucketBatchByLength operation on ds
@@ -331,9 +331,9 @@ TEST_F(MindDataTestPipeline, TestConcatFail1) {
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Rename operation on ds // Create a Rename operation on ds
@@ -362,7 +362,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Concat operation on the ds // Create a Concat operation on the ds
@@ -382,7 +382,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail3) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Concat operation on the ds // Create a Concat operation on the ds
@@ -402,7 +402,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail4) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Concat operation on the ds // Create a Concat operation on the ds
@@ -422,10 +422,10 @@ TEST_F(MindDataTestPipeline, TestConcatFail5) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -451,13 +451,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) {
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
folder_path = datasets_root_path_ + "/testCifar10Data/"; folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -502,13 +502,13 @@ TEST_F(MindDataTestPipeline, TestConcatGetDatasetSize) {
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
folder_path = datasets_root_path_ + "/testCifar10Data/"; folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -531,13 +531,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
// Column names: {"image", "label"} // Column names: {"image", "label"}
folder_path = datasets_root_path_ + "/testCifar10Data/"; folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -755,7 +755,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -796,7 +796,7 @@ TEST_F(MindDataTestPipeline, TestPipelineGetDatasetSize) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -818,7 +818,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<DistributedSampler>(4, 0, false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// num_per_shard is equal to 44/4 = 11 which is more than num_samples = 10, so the output is 10 // num_per_shard is equal to 44/4 = 11 which is more than num_samples = 10, so the output is 10
@@ -849,7 +849,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 15));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<DistributedSampler>(4, 0, false, 15));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// num_per_shard is equal to 44/4 = 11 which is less than num_samples = 15, so the output is 11 // num_per_shard is equal to 44/4 = 11 which is less than num_samples = 15, so the output is 11
@@ -879,7 +879,7 @@ TEST_F(MindDataTestPipeline, TestProjectMap) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -933,7 +933,7 @@ TEST_F(MindDataTestPipeline, TestProjectDuplicateColumnFail) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -962,7 +962,7 @@ TEST_F(MindDataTestPipeline, TestMapDuplicateColumnFail) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1002,7 +1002,7 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1054,7 +1054,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1078,7 +1078,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Rename operation on ds // Create a Rename operation on ds
@@ -1097,7 +1097,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail3) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Rename operation on ds // Create a Rename operation on ds
@@ -1124,7 +1124,7 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1174,7 +1174,7 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1217,7 +1217,7 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1256,7 +1256,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1276,7 +1276,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1295,7 +1295,7 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Shuffle operation on ds // Create a Shuffle operation on ds
@@ -1341,7 +1341,7 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Skip operation on ds // Create a Skip operation on ds
@@ -1379,7 +1379,7 @@ TEST_F(MindDataTestPipeline, TestSkipTakeRepeat) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 6));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 6));


// Create a Skip operation on ds // Create a Skip operation on ds
int32_t count = 0; int32_t count = 0;
@@ -1424,7 +1424,7 @@ TEST_F(MindDataTestPipeline, TestSkipGetDatasetSize) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Skip operation on ds // Create a Skip operation on ds
@@ -1440,7 +1440,7 @@ TEST_F(MindDataTestPipeline, TestSkipDatasetError1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Skip operation on ds with invalid count input // Create a Skip operation on ds with invalid count input
@@ -1459,7 +1459,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Take operation on ds, default count = -1 // Create a Take operation on ds, default count = -1
@@ -1496,7 +1496,7 @@ TEST_F(MindDataTestPipeline, TestTakeGetDatasetSize) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Take operation on ds, default count = -1 // Create a Take operation on ds, default count = -1
@@ -1511,7 +1511,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetError1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Take operation on ds with invalid count input // Create a Take operation on ds with invalid count input
@@ -1540,7 +1540,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 8));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 8));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Take operation on ds // Create a Take operation on ds
@@ -1577,7 +1577,7 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1631,11 +1631,11 @@ TEST_F(MindDataTestPipeline, TestZipFail) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Create a Zip operation on the datasets // Create a Zip operation on the datasets
@@ -1659,7 +1659,7 @@ TEST_F(MindDataTestPipeline, TestZipFail2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Zip operation on the datasets // Create a Zip operation on the datasets
@@ -1679,7 +1679,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -1688,7 +1688,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names)
@@ -1696,7 +1696,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


folder_path = datasets_root_path_ + "/testCifar10Data/"; folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -1749,7 +1749,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -1758,7 +1758,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) {
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 3));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names)
@@ -1766,7 +1766,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) {
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


folder_path = datasets_root_path_ + "/testCifar10Data/"; folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 5));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Project operation on ds // Create a Project operation on ds
@@ -1787,9 +1787,9 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 9));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 9));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names) // Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names)
@@ -1840,7 +1840,7 @@ TEST_F(MindDataTestPipeline, TestNumWorkersValidate) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 1));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 1));


// ds needs to be non nullptr otherwise, the subsequent logic will core dump // ds needs to be non nullptr otherwise, the subsequent logic will core dump
ASSERT_NE(ds, nullptr); ASSERT_NE(ds, nullptr);


+ 4
- 4
tests/ut/cpp/dataset/c_api_dataset_save.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) {
// Stage 1: load original dataset // Stage 1: load original dataset
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(0, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(0, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -70,7 +70,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) {


// Stage 3: Load dataset from file output by stage 2 // Stage 3: Load dataset from file output by stage 2
// Create a MindData Dataset // Create a MindData Dataset
std::shared_ptr<Dataset> ds_minddata = MindData(temp_file, {}, SequentialSampler(0, 10));
std::shared_ptr<Dataset> ds_minddata = MindData(temp_file, {}, std::make_shared<SequentialSampler>(0, 10));


// Create objects for the tensor ops // Create objects for the tensor ops
// uint32 will be casted to int64 implicitly in mindrecord file, so we have to cast it back to uint32 // uint32 will be casted to int64 implicitly in mindrecord file, so we have to cast it back to uint32
@@ -119,7 +119,7 @@ TEST_F(MindDataTestPipeline, TestSaveFail) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(0, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(0, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// fail with invalid dataset_path // fail with invalid dataset_path


+ 9
- 9
tests/ut/cpp/dataset/c_api_dataset_voc_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
class_index["cat"] = 1; class_index["cat"] = 1;
class_index["train"] = 9; class_index["train"] = 9;


std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetClassIndex) {
class_index["cat"] = 1; class_index["cat"] = 1;
class_index["train"] = 9; class_index["train"] = 9;


std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
@@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetters) {
class_index["cat"] = 1; class_index["cat"] = 1;
class_index["train"] = 9; class_index["train"] = 9;


std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


ds = ds->Batch(2); ds = ds->Batch(2);
@@ -120,7 +120,7 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {


// Create a VOC Dataset // Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) {


// Create a VOC Dataset // Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds1 = VOC(folder_path, "Classification", "train", {}, false, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds1, nullptr); EXPECT_NE(ds1, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) {
// Expect failure: invalid Manifest input, invalid task // Expect failure: invalid Manifest input, invalid task
EXPECT_EQ(iter1, nullptr); EXPECT_EQ(iter1, nullptr);


std::shared_ptr<Dataset> ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4));
std::shared_ptr<Dataset> ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds2, nullptr); EXPECT_NE(ds2, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -188,7 +188,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {


// Create a VOC Dataset // Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -240,7 +240,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentationError2) {
std::map<std::string, int32_t> class_index; std::map<std::string, int32_t> class_index;
class_index["car"] = 0; class_index["car"] = 0;
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, std::make_shared<RandomSampler>(false, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset


+ 8
- 8
tests/ut/cpp/dataset/c_api_datasets_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestCelebADataset) {


// Create a CelebA Dataset // Create a CelebA Dataset
std::string folder_path = datasets_root_path_ + "/testCelebAData/"; std::string folder_path = datasets_root_path_ + "/testCelebAData/";
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", SequentialSampler(0, 2), false, {});
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", std::make_shared<SequentialSampler>(0, 2), false, {});
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -208,7 +208,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongExtensionFail) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"});
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2), {".JGP"});
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -260,7 +260,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongSamplerFail) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(-2, 5));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(-2, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -274,7 +274,7 @@ TEST_F(MindDataTestPipeline, TestMnistGetDatasetSize) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 20); EXPECT_EQ(ds->GetDatasetSize(), 20);
} }
@@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMnistFailWithWrongDatasetDirFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFailWithWrongDatasetDirFail."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFailWithWrongDatasetDirFail.";


// Create a Mnist Dataset // Create a Mnist Dataset
std::shared_ptr<Dataset> ds = Mnist("", "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Mnist("", "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSize) {
std::map<std::string, int32_t> class_index; std::map<std::string, int32_t> class_index;
class_index["class1"] = 111; class_index["class1"] = 111;
class_index["class2"] = 333; class_index["class2"] = 333;
auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index);
auto ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, class_index);
EXPECT_EQ(ds->GetNumClasses(), 2); EXPECT_EQ(ds->GetNumClasses(), 2);
} }


@@ -320,6 +320,6 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSizeFail) {
std::map<std::string, int32_t> class_index; std::map<std::string, int32_t> class_index;
class_index["class1"] = 111; class_index["class1"] = 111;
class_index["wrong class"] = 333; class_index["wrong class"] = 333;
auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index);
auto ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, class_index);
EXPECT_EQ(ds->GetNumClasses(), -1); EXPECT_EQ(ds->GetNumClasses(), -1);
} }

+ 4
- 4
tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc View File

@@ -34,7 +34,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, sampler_size));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, sampler_size));
ds = ds->SetNumWorkers(2); ds = ds->SetNumWorkers(2);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -79,7 +79,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size));
ds = ds->SetNumWorkers(3); ds = ds->SetNumWorkers(3);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -125,7 +125,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size));
ds = ds->SetNumWorkers(3); ds = ds->SetNumWorkers(3);
ds = ds->Repeat(num_repeats); ds = ds->Repeat(num_repeats);


@@ -172,7 +172,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(5, sampler_size));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(5, sampler_size));
ds = ds->Repeat(num_repeats[0]); ds = ds->Repeat(num_repeats[0]);
ds = ds->Repeat(num_repeats[1]); ds = ds->Repeat(num_repeats[1]);




+ 168
- 51
tests/ut/cpp/dataset/c_api_samplers_test.cc View File

@@ -16,6 +16,7 @@
#include "common/common.h" #include "common/common.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
#include "minddata/dataset/include/datasets.h" #include "minddata/dataset/include/datasets.h"
#include <functional>


using namespace mindspore::dataset; using namespace mindspore::dataset;
using mindspore::dataset::Tensor; using mindspore::dataset::Tensor;
@@ -25,27 +26,27 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
}; };


TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) {
std::shared_ptr<SamplerObj> sampl = DistributedSampler(2, 1);
std::shared_ptr<Sampler> sampl = std::make_shared<DistributedSampler>(2, 1);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


sampl = PKSampler(3);
sampl = std::make_shared<PKSampler>(3);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


sampl = RandomSampler(false, 12);
sampl = std::make_shared<RandomSampler>(false, 12);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


sampl = SequentialSampler(0, 12);
sampl = std::make_shared<SequentialSampler>(0, 12);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1};
sampl = WeightedRandomSampler(weights, 12);
sampl = std::make_shared<WeightedRandomSampler>(weights, 12);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23};
sampl = SubsetSampler(indices);
sampl = std::make_shared<SubsetSampler>(indices);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


sampl = SubsetRandomSampler(indices);
sampl = std::make_shared<SubsetRandomSampler>(indices);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
@@ -88,47 +89,47 @@ TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) {


TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
int64_t num_rows = 30; // dummy variable for number of rows in the dataset int64_t num_rows = 30; // dummy variable for number of rows in the dataset
std::shared_ptr<SamplerObj> sampl = DistributedSampler(2, 1, false, 6);
std::shared_ptr<SamplerObj> sampl = std::make_shared<DistributedSamplerObj>(2, 1, false, 6, 1, -1, true);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);
std::shared_ptr<SamplerRT> sampler_rt; std::shared_ptr<SamplerRT> sampler_rt;
sampl->SamplerBuild(&sampler_rt); sampl->SamplerBuild(&sampler_rt);
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 6); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 6);


sampl = PKSampler(3, false);
sampl = std::make_shared<PKSamplerObj>(3, false, 0);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);
sampl->SamplerBuild(&sampler_rt); sampl->SamplerBuild(&sampler_rt);
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 30); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 30);


sampl = RandomSampler(false, 12);
sampl = std::make_shared<RandomSamplerObj>(false, 12);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);
sampl->SamplerBuild(&sampler_rt); sampl->SamplerBuild(&sampler_rt);
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12);


sampl = SequentialSampler(0, 10);
sampl = std::make_shared<SequentialSamplerObj>(0, 10);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);
sampl->SamplerBuild(&sampler_rt); sampl->SamplerBuild(&sampler_rt);
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 10); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 10);


std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1};
sampl = WeightedRandomSampler(weights, 12);
sampl = std::make_shared<WeightedRandomSamplerObj>(weights, 12);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);
sampl->SamplerBuild(&sampler_rt); sampl->SamplerBuild(&sampler_rt);
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12);


std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21}; std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21};
sampl = SubsetRandomSampler(indices, 11);
sampl = std::make_shared<SubsetRandomSamplerObj>(indices, 11);
EXPECT_NE(sampl, nullptr); EXPECT_NE(sampl, nullptr);
sampl->SamplerBuild(&sampler_rt); sampl->SamplerBuild(&sampler_rt);
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 11); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 11);


// Testing chains // Testing chains
// Parent and child have num_samples // Parent and child have num_samples
std::shared_ptr<SamplerObj> sampl1 = WeightedRandomSampler(weights, 12);
std::shared_ptr<SamplerObj> sampl1 = std::make_shared<WeightedRandomSamplerObj>(weights, 12);
EXPECT_NE(sampl1, nullptr); EXPECT_NE(sampl1, nullptr);
std::shared_ptr<SamplerRT> sampler_rt1; std::shared_ptr<SamplerRT> sampler_rt1;
sampl1->SamplerBuild(&sampler_rt1); sampl1->SamplerBuild(&sampler_rt1);


std::shared_ptr<SamplerObj> sampl2 = SequentialSampler(0, 10);
std::shared_ptr<SamplerObj> sampl2 = std::make_shared<SequentialSamplerObj>(0, 10);
EXPECT_NE(sampl2, nullptr); EXPECT_NE(sampl2, nullptr);
std::shared_ptr<SamplerRT> sampler_rt2; std::shared_ptr<SamplerRT> sampler_rt2;
sampl2->SamplerBuild(&sampler_rt2); sampl2->SamplerBuild(&sampler_rt2);
@@ -136,12 +137,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
EXPECT_EQ(sampler_rt2->CalculateNumSamples(num_rows), 10); EXPECT_EQ(sampler_rt2->CalculateNumSamples(num_rows), 10);


// Parent doesn't have num_samples // Parent doesn't have num_samples
std::shared_ptr<SamplerObj> sampl3 = WeightedRandomSampler(weights, 12);
std::shared_ptr<SamplerObj> sampl3 = std::make_shared<WeightedRandomSamplerObj>(weights, 12);
EXPECT_NE(sampl3, nullptr); EXPECT_NE(sampl3, nullptr);
std::shared_ptr<SamplerRT> sampler_rt3; std::shared_ptr<SamplerRT> sampler_rt3;
sampl3->SamplerBuild(&sampler_rt3); sampl3->SamplerBuild(&sampler_rt3);


std::shared_ptr<SamplerObj> sampl4 = SubsetRandomSampler(indices);
std::shared_ptr<SamplerObj> sampl4 = std::make_shared<SubsetRandomSamplerObj>(indices, 0);
EXPECT_NE(sampl4, nullptr); EXPECT_NE(sampl4, nullptr);
std::shared_ptr<SamplerRT> sampler_rt4; std::shared_ptr<SamplerRT> sampler_rt4;
sampl4->SamplerBuild(&sampler_rt4); sampl4->SamplerBuild(&sampler_rt4);
@@ -149,12 +150,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
EXPECT_EQ(sampler_rt4->CalculateNumSamples(num_rows), 11); EXPECT_EQ(sampler_rt4->CalculateNumSamples(num_rows), 11);


// Child doesn't have num_samples // Child doesn't have num_samples
std::shared_ptr<SamplerObj> sampl5 = RandomSampler(false);
std::shared_ptr<SamplerObj> sampl5 = std::make_shared<RandomSamplerObj>(false, 0);
EXPECT_NE(sampl5, nullptr); EXPECT_NE(sampl5, nullptr);
std::shared_ptr<SamplerRT> sampler_rt5; std::shared_ptr<SamplerRT> sampler_rt5;
sampl5->SamplerBuild(&sampler_rt5); sampl5->SamplerBuild(&sampler_rt5);


std::shared_ptr<SamplerObj> sampl6 = PKSampler(3, false, 7);
std::shared_ptr<SamplerObj> sampl6 = std::make_shared<PKSamplerObj>(3, false, 7);
EXPECT_NE(sampl6, nullptr); EXPECT_NE(sampl6, nullptr);
std::shared_ptr<SamplerRT> sampler_rt6; std::shared_ptr<SamplerRT> sampler_rt6;
sampl6->SamplerBuild(&sampler_rt6); sampl6->SamplerBuild(&sampler_rt6);
@@ -164,41 +165,80 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {


TEST_F(MindDataTestPipeline, TestSamplersMoveParameters) { TEST_F(MindDataTestPipeline, TestSamplersMoveParameters) {
std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23};
std::shared_ptr<SamplerObj> sampl1 = SubsetRandomSampler(indices);
std::shared_ptr<SamplerObj> sampl1 = std::make_shared<SubsetRandomSamplerObj>(indices, 0);
EXPECT_FALSE(indices.empty()); EXPECT_FALSE(indices.empty());
std::shared_ptr<SamplerRT> sampler_rt = nullptr; std::shared_ptr<SamplerRT> sampler_rt = nullptr;
sampl1->SamplerBuild(&sampler_rt); sampl1->SamplerBuild(&sampler_rt);
EXPECT_NE(sampler_rt, nullptr); EXPECT_NE(sampler_rt, nullptr);
std::shared_ptr<SamplerObj> sampl2 = SubsetRandomSampler(std::move(indices));
std::shared_ptr<SamplerObj> sampl2 = std::make_shared<SubsetRandomSamplerObj>(std::move(indices), 0);
EXPECT_TRUE(indices.empty()); EXPECT_TRUE(indices.empty());
std::shared_ptr<SamplerRT> sampler_rt2 = nullptr; std::shared_ptr<SamplerRT> sampler_rt2 = nullptr;
sampl2->SamplerBuild(&sampler_rt2); sampl2->SamplerBuild(&sampler_rt2);
EXPECT_NE(sampler_rt, nullptr); EXPECT_NE(sampler_rt, nullptr);
} }


TEST_F(MindDataTestPipeline, TestWeightedRandomSamplerFail) {
// weights is empty
std::vector<double> weights1 = {};
std::shared_ptr<SamplerObj> sampl1 = WeightedRandomSampler(weights1);
EXPECT_EQ(sampl1, nullptr);

// weights has negative number
std::vector<double> weights2 = {0.5, 0.2, -0.4};
std::shared_ptr<SamplerObj> sampl2 = WeightedRandomSampler(weights2);
EXPECT_EQ(sampl2, nullptr);

// weights elements are all zero
std::vector<double> weights3 = {0, 0, 0};
std::shared_ptr<SamplerObj> sampl3 = WeightedRandomSampler(weights3);
EXPECT_EQ(sampl3, nullptr);
TEST_F(MindDataTestPipeline, TestNoSamplerSuccess1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNoSamplerSuccess1.";
// Test building a dataset with no sampler provided (defaults to random sampler

// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false);
EXPECT_NE(ds, nullptr);

// Iterate the dataset and get each row
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

uint64_t i = 0;
while (row.size() != 0) {
i++;
auto label = row["label"];
iter->GetNextRow(&row);
}

EXPECT_EQ(i, ds->GetDatasetSize());
iter->Stop();
}

TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess1.";
// Test basic setting of distributed_sampler

// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true
std::shared_ptr<Sampler> sampler = std::make_shared<DistributedSampler>(4, 0, false, 0, 0, -1, true);
EXPECT_NE(sampler, nullptr);

// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
EXPECT_NE(ds, nullptr);

// Iterate the dataset and get each row
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

uint64_t i = 0;
while (row.size() != 0) {
i++;
auto label = row["label"];
iter->GetNextRow(&row);
}

EXPECT_EQ(i, 11);
iter->Stop();
} }


TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess.";
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess2.";
// Test basic setting of distributed_sampler // Test basic setting of distributed_sampler


// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true
std::shared_ptr<SamplerObj> sampler = DistributedSampler(4, 0, false, 0, 0, -1, true);
Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, -1, true);
EXPECT_NE(sampler, nullptr); EXPECT_NE(sampler, nullptr);


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
@@ -223,16 +263,102 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) {
iter->Stop(); iter->Stop();
} }


TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess3.";
// Test basic setting of distributed_sampler

// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true
DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, -1, true);

// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
EXPECT_NE(ds, nullptr);

// Iterate the dataset and get each row
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

uint64_t i = 0;
while (row.size() != 0) {
i++;
auto label = row["label"];
iter->GetNextRow(&row);
}

EXPECT_EQ(i, 11);
iter->Stop();
}

TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1.";
// Test basic setting of distributed_sampler

// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true
// offset=5 which is greater than num_shards=4 --> will fail later
std::shared_ptr<Sampler> sampler = std::make_shared<DistributedSampler>(4, 0, false, 0, 0, 5, false);
EXPECT_NE(sampler, nullptr);

// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
EXPECT_NE(ds, nullptr);

// Iterate will fail because sampler is not initiated successfully.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_EQ(iter, nullptr);
}

TEST_F(MindDataTestPipeline, TestDistributedSamplerFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail2.";
// Test basic setting of distributed_sampler

// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true
// offset=5 which is greater than num_shards=4 --> will fail later
Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, 5, false);
EXPECT_NE(sampler, nullptr);

// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
EXPECT_NE(ds, nullptr);

// Iterate will fail because sampler is not initiated successfully.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_EQ(iter, nullptr);
}

TEST_F(MindDataTestPipeline, TestDistributedSamplerFail3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail3.";
// Test basic setting of distributed_sampler

// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true
// offset=5 which is greater than num_shards=4 --> will fail later
DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, 5, false);

// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
EXPECT_NE(ds, nullptr);

// Iterate will fail because sampler is not initiated successfully.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_EQ(iter, nullptr);
}

TEST_F(MindDataTestPipeline, TestSamplerAddChild) { TEST_F(MindDataTestPipeline, TestSamplerAddChild) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSamplerAddChild."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSamplerAddChild.";


auto sampler = DistributedSampler(1, 0, false, 5, 0, -1, true);
auto sampler = std::make_shared<DistributedSampler>(1, 0, false, 5, 0, -1, true);
EXPECT_NE(sampler, nullptr); EXPECT_NE(sampler, nullptr);


auto child_sampler = SequentialSampler();
sampler->AddChildSampler(child_sampler);
auto child_sampler = std::make_shared<SequentialSampler>();
EXPECT_NE(child_sampler, nullptr); EXPECT_NE(child_sampler, nullptr);


sampler->AddChild(child_sampler);

// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
@@ -253,12 +379,3 @@ TEST_F(MindDataTestPipeline, TestSamplerAddChild) {
EXPECT_EQ(ds->GetDatasetSize(), 5); EXPECT_EQ(ds->GetDatasetSize(), 5);
iter->Stop(); iter->Stop();
} }

TEST_F(MindDataTestPipeline, TestDistributedSamplerFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail.";
// Test invalid offset setting of distributed_sampler

// offset=5 which is greater than num_shards=4
std::shared_ptr<SamplerObj> sampler = DistributedSampler(4, 0, false, 0, 0, 5, false);
EXPECT_EQ(sampler, nullptr);
}

+ 22
- 20
tests/ut/cpp/dataset/c_api_transforms_test.cc View File

@@ -33,7 +33,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 3));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -68,6 +68,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {


EXPECT_EQ(i, 3); EXPECT_EQ(i, 3);


// Manually terminate the pipeline // Manually terminate the pipeline
iter->Stop(); iter->Stop();
} }
@@ -77,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail1) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Resize: Non-positive size value: -1 at element: 0 // Resize: Non-positive size value: -1 at element: 0
@@ -100,7 +101,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail2) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Compose: transform ops must not be null // Compose: transform ops must not be null
@@ -121,7 +122,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail3) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Compose: transform list must not be empty // Compose: transform list must not be empty
@@ -142,7 +143,7 @@ TEST_F(MindDataTestPipeline, TestDuplicateSuccess) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -182,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) {
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
int number_of_classes = 10; int number_of_classes = 10;
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -248,7 +249,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) {
TEST_F(MindDataTestPipeline, TestOneHotSuccess2) { TEST_F(MindDataTestPipeline, TestOneHotSuccess2) {
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -299,7 +300,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail1) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// incorrect num_class // incorrect num_class
@@ -320,7 +321,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail2) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// incorrect num_class // incorrect num_class
@@ -341,7 +342,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -382,7 +383,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail1) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Resize: Non-positive size value: -1 at element: 0 // Resize: Non-positive size value: -1 at element: 0
@@ -405,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail2) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// RandomApply: transform ops must not be null // RandomApply: transform ops must not be null
@@ -426,7 +427,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail3) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// RandomApply: Probability has to be between 0 and 1 // RandomApply: Probability has to be between 0 and 1
@@ -447,7 +448,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail4) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// RandomApply: transform list must not be empty // RandomApply: transform list must not be empty
@@ -468,7 +469,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -510,7 +511,8 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
RandomSampler sampler = RandomSampler(false, 10);
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", sampler);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Resize: Non-positive size value: -1 at element: 0 // Resize: Non-positive size value: -1 at element: 0
@@ -533,7 +535,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// RandomChoice: transform ops must not be null // RandomChoice: transform ops must not be null
@@ -554,7 +556,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// RandomChoice: transform list must not be empty // RandomChoice: transform list must not be empty
@@ -575,7 +577,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastSuccess) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 1));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 1));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -625,7 +627,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastFail) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// incorrect data type // incorrect data type


+ 20
- 20
tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc View File

@@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -142,7 +142,7 @@ TEST_F(MindDataTestPipeline, TestCenterCrop) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -228,7 +228,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) {
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
int number_of_classes = 10; int number_of_classes = 10;
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess2) {
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
int number_of_classes = 10; int number_of_classes = 10;
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -358,7 +358,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail1) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -393,7 +393,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail2) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -428,7 +428,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail3) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -462,7 +462,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter.";


std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -521,7 +521,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestDecode) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -620,7 +620,7 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -672,7 +672,7 @@ TEST_F(MindDataTestPipeline, TestInvert) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 20));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 20));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -710,7 +710,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -744,7 +744,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail2) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -778,7 +778,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -829,7 +829,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {


// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Batch operation on ds // Create a Batch operation on ds
@@ -880,7 +880,7 @@ TEST_F(MindDataTestPipeline, TestNormalize) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -958,7 +958,7 @@ TEST_F(MindDataTestPipeline, TestNormalizePad) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1025,7 +1025,7 @@ TEST_F(MindDataTestPipeline, TestPad) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds


+ 1
- 1
tests/ut/cpp/dataset/c_api_vision_bounding_box_augment_test.cc View File

@@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestBoundingBoxAugmentSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBoundingBoxAugmentSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBoundingBoxAugmentSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


/* FIXME - Resolve BoundingBoxAugment to properly handle TensorTransform input /* FIXME - Resolve BoundingBoxAugment to properly handle TensorTransform input


+ 4
- 4
tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc View File

@@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess1."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess1.";
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, 1));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 1));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset
@@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess2 with different params."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess2 with different params.";
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 1));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -121,7 +121,7 @@ TEST_F(MindDataTestPipeline, TestResize1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResize1 with single integer input."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResize1 with single integer input.";
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 6));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -183,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestResizeWithBBoxSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResizeWithBBoxSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResizeWithBBoxSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops


+ 1
- 1
tests/ut/cpp/dataset/c_api_vision_random_subselect_policy_test.cc View File

@@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestRandomSelectSubpolicySuccess) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


/* FIXME - Resolve RandomSelectSubpolicy to properly handle TensorTransform input /* FIXME - Resolve RandomSelectSubpolicy to properly handle TensorTransform input


+ 36
- 36
tests/ut/cpp/dataset/c_api_vision_random_test.cc View File

@@ -46,7 +46,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -96,7 +96,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -145,7 +145,7 @@ TEST_F(MindDataTestPipeline, TestRandomColor) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -209,7 +209,7 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -303,7 +303,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 10));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -366,7 +366,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropFail) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -406,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropWithBboxSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropWithBboxSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -446,7 +446,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxFail) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -493,7 +493,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomHorizontalFlipWithBBoxSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomHorizontalFlipWithBBoxSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -531,7 +531,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxFail) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -548,7 +548,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalAndVerticalFlip) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -618,7 +618,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -667,7 +667,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -716,7 +716,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -756,7 +756,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -821,7 +821,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess1) {


// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -861,7 +861,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess2) {


// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -921,7 +921,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess1) {
// Testing RandomResizedCrop with default values // Testing RandomResizedCrop with default values
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -960,7 +960,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess2) {
// Testing RandomResizedCrop with non-default values // Testing RandomResizedCrop with non-default values
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1000,7 +1000,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail1) {
// This should fail because size has negative value // This should fail because size has negative value
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1020,7 +1020,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail2) {
// This should fail because scale isn't in {min, max} format // This should fail because scale isn't in {min, max} format
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1040,7 +1040,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail3) {
// This should fail because ratio isn't in {min, max} format // This should fail because ratio isn't in {min, max} format
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1060,7 +1060,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail4) {
// This should fail because scale has a size of more than 2 // This should fail because scale has a size of more than 2
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1080,7 +1080,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess1) {
// Testing RandomResizedCropWithBBox with default values // Testing RandomResizedCropWithBBox with default values
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1119,7 +1119,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess2) {
// Testing RandomResizedCropWithBBox with non-default values // Testing RandomResizedCropWithBBox with non-default values
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1160,7 +1160,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail1) {
// This should fail because size has negative value // This should fail because size has negative value
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1173,7 +1173,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail2) {
// This should fail because scale isn't in {min, max} format // This should fail because scale isn't in {min, max} format
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1186,7 +1186,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail3) {
// This should fail because ratio isn't in {min, max} format // This should fail because ratio isn't in {min, max} format
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1199,7 +1199,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail4) {
// This should fail because scale has a size of more than 2 // This should fail because scale has a size of more than 2
// Create a Cifar10 Dataset // Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1212,7 +1212,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotation) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1273,7 +1273,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotationFail) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1309,7 +1309,7 @@ TEST_F(MindDataTestPipeline, TestRandomSharpness) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -1374,7 +1374,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1414,7 +1414,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1484,7 +1484,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomVerticalFlipWithBBoxSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomVerticalFlipWithBBoxSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -1522,7 +1522,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxFail) {
// FIXME: For error tests, need to check for failure from CreateIterator execution // FIXME: For error tests, need to check for failure from CreateIterator execution
// Create an VOC Dataset // Create an VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops


+ 4
- 4
tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc View File

@@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess1) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 4));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -74,7 +74,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess2) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 6));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 6));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops
@@ -158,7 +158,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess1 with single integer input."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess1 with single integer input.";
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 4));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds
@@ -201,7 +201,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess2 with (height, width) input."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess2 with (height, width) input.";
// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 2));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create SoftDvppDecodeResizeJpeg object with single integer input // Create SoftDvppDecodeResizeJpeg object with single integer input


+ 1
- 1
tests/ut/cpp/dataset/c_api_vision_uniform_aug_test.cc View File

@@ -78,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create a Repeat operation on ds // Create a Repeat operation on ds


+ 2
- 2
tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc View File

@@ -36,7 +36,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeDisabled) {
MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeDisabled"; MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeDisabled";


std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));


// Create objects for the tensor ops // Create objects for the tensor ops
std::shared_ptr<TensorTransform> decode(new vision::Decode()); std::shared_ptr<TensorTransform> decode(new vision::Decode());
@@ -67,7 +67,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeEnabled) {
MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeEnabled"; MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeEnabled";


std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));


// Create objects for the tensor ops // Create objects for the tensor ops
std::shared_ptr<TensorTransform> decode(new vision::Decode()); std::shared_ptr<TensorTransform> decode(new vision::Decode());


+ 4
- 4
tests/ut/cpp/dataset/ir_tree_adapter_test.cc View File

@@ -1,5 +1,5 @@
/** /**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -48,7 +48,7 @@ TEST_F(MindDataTestTreeAdapter, TestSimpleTreeAdapter) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", SequentialSampler(0, 4));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 4));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


ds = ds->Batch(2); ds = ds->Batch(2);
@@ -83,7 +83,7 @@ TEST_F(MindDataTestTreeAdapter, TestTreeAdapterWithRepeat) {


// Create a Mnist Dataset // Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/"; std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", SequentialSampler(0, 3));
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 3));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


ds = ds->Batch(2, false); ds = ds->Batch(2, false);
@@ -115,7 +115,7 @@ TEST_F(MindDataTestTreeAdapter, TestProjectMapTreeAdapter) {


// Create an ImageFolder Dataset // Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, 2));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);


// Create objects for the tensor ops // Create objects for the tensor ops


+ 38
- 38
tests/ut/cpp/dataset/tree_modifying_function_test.cc View File

@@ -49,9 +49,9 @@ TEST_F(MindDataTestTreeModifying, AppendChild) {
* ds1 * ds1
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds3 = ds1->Take(10); std::shared_ptr<Dataset> ds3 = ds1->Take(10);
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
Status rc; Status rc;
@@ -105,9 +105,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds3 = ds1->Take(10); std::shared_ptr<Dataset> ds3 = ds1->Take(10);
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ds5->Repeat(4); std::shared_ptr<Dataset> ds2 = ds5->Repeat(4);
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
Status rc; Status rc;
@@ -119,7 +119,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
EXPECT_EQ(rc, Status::OK()); EXPECT_EQ(rc, Status::OK());
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode(); std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode();
std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1]; std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1];
rc = ds4_node->InsertChildAt(1, ds6_to_insert); rc = ds4_node->InsertChildAt(1, ds6_to_insert);
@@ -132,7 +132,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
EXPECT_EQ(rc, Status::OK()); EXPECT_EQ(rc, Status::OK());
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
ds6_to_insert = ds6->IRNode(); ds6_to_insert = ds6->IRNode();
std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0]; std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0];
rc = ds4_node->InsertChildAt(0, ds6_to_insert); rc = ds4_node->InsertChildAt(0, ds6_to_insert);
@@ -145,7 +145,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
EXPECT_EQ(rc, Status::OK()); EXPECT_EQ(rc, Status::OK());
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
ds6_to_insert = ds6->IRNode(); ds6_to_insert = ds6->IRNode();
rc = ds4_node->InsertChildAt(2, ds6_to_insert); rc = ds4_node->InsertChildAt(2, ds6_to_insert);
EXPECT_EQ(rc, Status::OK()); EXPECT_EQ(rc, Status::OK());
@@ -165,9 +165,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds3 = ds1->Take(10); std::shared_ptr<Dataset> ds3 = ds1->Take(10);
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ds5->Repeat(4); std::shared_ptr<Dataset> ds2 = ds5->Repeat(4);
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
Status rc; Status rc;
@@ -179,7 +179,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) {
EXPECT_EQ(rc, Status::OK()); EXPECT_EQ(rc, Status::OK());
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode(); std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode();
std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0]; std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0];
std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1]; std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1];
@@ -193,7 +193,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) {
EXPECT_EQ(rc, Status::OK()); EXPECT_EQ(rc, Status::OK());
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
ds6_to_insert = ds6->IRNode(); ds6_to_insert = ds6->IRNode();
ds3_node = ds4_node->Children()[0]; ds3_node = ds4_node->Children()[0];
ds2_node = ds4_node->Children()[1]; ds2_node = ds4_node->Children()[1];
@@ -246,8 +246,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove01) {
*/ */
// Case 1 // Case 1
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds3 = ds1->Take(10); std::shared_ptr<Dataset> ds3 = ds1->Take(10);
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
Status rc; Status rc;
@@ -271,8 +271,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove02) {


// Case 2 // Case 2
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds3 = ds1->Take(10); std::shared_ptr<Dataset> ds3 = ds1->Take(10);
std::shared_ptr<Dataset> ds4 = ds3 + ds2; std::shared_ptr<Dataset> ds4 = ds3 + ds2;
Status rc; Status rc;
@@ -296,8 +296,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove03) {


// Case 3 // Case 3
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds3 = ds1->Take(10); std::shared_ptr<Dataset> ds3 = ds1->Take(10);
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
Status rc; Status rc;
@@ -357,11 +357,11 @@ TEST_F(MindDataTestTreeModifying, Drop01) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds8 = ds7->Take(20); std::shared_ptr<Dataset> ds8 = ds7->Take(20);
std::shared_ptr<Dataset> ds9 = ds8->Skip(1); std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
std::shared_ptr<Dataset> ds6 = ds4->Take(13); std::shared_ptr<Dataset> ds6 = ds4->Take(13);
std::shared_ptr<Dataset> ds10 = ds9 + ds6; std::shared_ptr<Dataset> ds10 = ds9 + ds6;
@@ -419,11 +419,11 @@ TEST_F(MindDataTestTreeModifying, Drop03) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds8 = ds7->Take(20); std::shared_ptr<Dataset> ds8 = ds7->Take(20);
std::shared_ptr<Dataset> ds9 = ds8->Skip(1); std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
std::shared_ptr<Dataset> ds6 = ds4->Take(13); std::shared_ptr<Dataset> ds6 = ds4->Take(13);
std::shared_ptr<Dataset> ds10 = ds9 + ds6; std::shared_ptr<Dataset> ds10 = ds9 + ds6;
@@ -469,14 +469,14 @@ TEST_F(MindDataTestTreeModifying, Drop04) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds8 = ds7->Take(20); std::shared_ptr<Dataset> ds8 = ds7->Take(20);
std::shared_ptr<Dataset> ds9 = ds8->Skip(1); std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1});
std::shared_ptr<Dataset> ds10 = ds9 + ds6; std::shared_ptr<Dataset> ds10 = ds9 + ds6;
Status rc; Status rc;
@@ -526,13 +526,13 @@ TEST_F(MindDataTestTreeModifying, Drop05) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds8 = ds7->Take(20); std::shared_ptr<Dataset> ds8 = ds7->Take(20);
std::shared_ptr<Dataset> ds9 = ds8->Skip(1); std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds4 = ds3->Skip(1); std::shared_ptr<Dataset> ds4 = ds3->Skip(1);
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1});
std::shared_ptr<Dataset> ds10 = ds9 + ds6; std::shared_ptr<Dataset> ds10 = ds9 + ds6;
Status rc; Status rc;
@@ -581,14 +581,14 @@ TEST_F(MindDataTestTreeModifying, Drop06) {
* *
*/ */
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds8 = ds7->Take(20); std::shared_ptr<Dataset> ds8 = ds7->Take(20);
std::shared_ptr<Dataset> ds9 = ds8->Skip(1); std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); // ds1 is put after (ds5, ds4)!!! std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); // ds1 is put after (ds5, ds4)!!!
std::shared_ptr<Dataset> ds10 = ds9 + ds6; std::shared_ptr<Dataset> ds10 = ds9 + ds6;
Status rc; Status rc;


Loading…
Cancel
Save