From: @mhmotallebi Reviewed-by: Signed-off-by:tags/v1.2.0-rc1
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -285,7 +285,24 @@ std::shared_ptr<SchemaObj> Schema(const std::string &schema_file) { | |||||
| // Function to create a AlbumDataset. | // Function to create a AlbumDataset. | ||||
| std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | ||||
| const std::vector<std::string> &column_names, bool decode, | const std::vector<std::string> &column_names, bool decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a AlbumDataset. | |||||
| std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a AlbumDataset. | |||||
| std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache); | auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache); | ||||
| @@ -295,7 +312,24 @@ std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::s | |||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| // Function to create a CelebADataset. | // Function to create a CelebADataset. | ||||
| std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, | std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, bool decode, | |||||
| const std::shared_ptr<Sampler> &sampler, bool decode, | |||||
| const std::set<std::string> &extensions, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a CelebADataset. | |||||
| std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| bool decode, const std::set<std::string> &extensions, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a CelebADataset. | |||||
| std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, bool decode, | |||||
| const std::set<std::string> &extensions, | const std::set<std::string> &extensions, | ||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache); | auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache); | ||||
| @@ -305,7 +339,22 @@ std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std: | |||||
| // Function to create a Cifar10Dataset. | // Function to create a Cifar10Dataset. | ||||
| std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, | std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a Cifar10Dataset. | |||||
| std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a Cifar10Dataset. | |||||
| std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache); | auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache); | ||||
| @@ -314,7 +363,22 @@ std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const st | |||||
| // Function to create a Cifar100Dataset. | // Function to create a Cifar100Dataset. | ||||
| std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, | std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a Cifar100Dataset. | |||||
| std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a Cifar100Dataset. | |||||
| std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache); | auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache); | ||||
| @@ -330,10 +394,26 @@ std::shared_ptr<CLUEDataset> CLUE(const std::vector<std::string> &clue_files, co | |||||
| return ds; | return ds; | ||||
| } | } | ||||
| // Function to create a CocoDataset. | |||||
| std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | |||||
| const std::string &task, const bool &decode, const std::shared_ptr<Sampler> &sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a CocoDataset. | |||||
| std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | |||||
| const std::string &task, const bool &decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a CocoDataset. | // Function to create a CocoDataset. | ||||
| std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | ||||
| const std::string &task, const bool &decode, | const std::string &task, const bool &decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache); | auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache); | ||||
| @@ -353,7 +433,16 @@ std::shared_ptr<CSVDataset> CSV(const std::vector<std::string> &dataset_files, c | |||||
| // Function to create a ImageFolderDataset. | // Function to create a ImageFolderDataset. | ||||
| std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, | std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::set<std::string> &extensions, | |||||
| const std::map<std::string, int32_t> &class_indexing, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<ImageFolderDataset>(dataset_dir, decode, sampler, extensions, class_indexing, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a ImageFolderDataset. | |||||
| std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler, | |||||
| const std::set<std::string> &extensions, | const std::set<std::string> &extensions, | ||||
| const std::map<std::string, int32_t> &class_indexing, | const std::map<std::string, int32_t> &class_indexing, | ||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| @@ -361,10 +450,37 @@ std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, | |||||
| return ds; | return ds; | ||||
| } | } | ||||
| // Function to create a ImageFolderDataset. | |||||
| std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::set<std::string> &extensions, | |||||
| const std::map<std::string, int32_t> &class_indexing, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<ImageFolderDataset>(dataset_dir, decode, sampler, extensions, class_indexing, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a ManifestDataset. | |||||
| std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a ManifestDataset. | |||||
| std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a ManifestDataset. | // Function to create a ManifestDataset. | ||||
| std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, | std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | const std::map<std::string, int32_t> &class_indexing, bool decode, | ||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache); | auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache); | ||||
| @@ -374,7 +490,22 @@ std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const | |||||
| // Function to create a MindDataDataset. | // Function to create a MindDataDataset. | ||||
| std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, | std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, | ||||
| const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample, | |||||
| const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | |||||
| auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a MindDataDataset. | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) { | |||||
| auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a MindDataDataset. | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded); | auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded); | ||||
| @@ -384,7 +515,24 @@ std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const | |||||
| // Function to create a MindDataDataset. | // Function to create a MindDataDataset. | ||||
| std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | ||||
| const std::vector<std::string> &columns_list, | const std::vector<std::string> &columns_list, | ||||
| const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample, | |||||
| const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | |||||
| auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a MindDataDataset. | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | |||||
| const std::vector<std::string> &columns_list, Sampler *sampler, | |||||
| nlohmann::json padded_sample, int64_t num_padded) { | |||||
| auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a MindDataDataset. | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | |||||
| const std::vector<std::string> &columns_list, | |||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded); | auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded); | ||||
| @@ -393,7 +541,22 @@ std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &datase | |||||
| #endif | #endif | ||||
| // Function to create a MnistDataset. | // Function to create a MnistDataset. | ||||
| std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, | std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a MnistDataset. | |||||
| std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a MnistDataset. | |||||
| std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache); | auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache); | ||||
| @@ -419,7 +582,23 @@ std::shared_ptr<TextFileDataset> TextFile(const std::vector<std::string> &datase | |||||
| // Function to create a VOCDataset. | // Function to create a VOCDataset. | ||||
| std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, | std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, | ||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | const std::map<std::string, int32_t> &class_indexing, bool decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a VOCDataset. | |||||
| std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache); | |||||
| return ds; | |||||
| } | |||||
| // Function to create a VOCDataset. | |||||
| std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache); | auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache); | ||||
| @@ -831,41 +1010,117 @@ std::shared_ptr<DatasetCache> CreateDatasetCache(session_id_type id, uint64_t me | |||||
| AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | ||||
| const std::vector<std::string> &column_names, bool decode, | const std::vector<std::string> &column_names, bool decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler, cache); | |||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, | CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, bool decode, | |||||
| const std::shared_ptr<Sampler> &sampler, bool decode, | |||||
| const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, bool decode, | |||||
| const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, bool decode, | |||||
| const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) { | const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler, decode, extensions, cache); | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, | Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler, cache); | |||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, | Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler, cache); | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| CLUEDataset::CLUEDataset(const std::vector<std::string> &dataset_files, const std::string &task, | CLUEDataset::CLUEDataset(const std::vector<std::string> &dataset_files, const std::string &task, | ||||
| const std::string &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, | const std::string &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, | ||||
| int32_t shard_id, const std::shared_ptr<DatasetCache> &cache) { | int32_t shard_id, const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<CLUENode>(dataset_files, task, usage, num_samples, shuffle, num_shards, shard_id, cache); | auto ds = std::make_shared<CLUENode>(dataset_files, task, usage, num_samples, shuffle, num_shards, shard_id, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, | |||||
| const bool &decode, const std::shared_ptr<Sampler> &sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, | |||||
| const bool &decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, | CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, | ||||
| const bool &decode, const std::shared_ptr<SamplerObj> &sampler, | |||||
| const bool &decode, const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler, cache); | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field_delim, | CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field_delim, | ||||
| const std::vector<std::shared_ptr<CsvBase>> &column_defaults, | const std::vector<std::shared_ptr<CsvBase>> &column_defaults, | ||||
| const std::vector<std::string> &column_names, int64_t num_samples, ShuffleMode shuffle, | const std::vector<std::string> &column_names, int64_t num_samples, ShuffleMode shuffle, | ||||
| @@ -874,8 +1129,21 @@ CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field | |||||
| num_shards, shard_id, cache); | num_shards, shard_id, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, | ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::shared_ptr<Sampler> &sampler, const std::set<std::string> &extensions, | |||||
| const std::map<std::string, int32_t> &class_indexing, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| // This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false. | |||||
| bool recursive = false; | |||||
| // Create logical representation of ImageFolderDataset. | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = | |||||
| std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler, | |||||
| const std::set<std::string> &extensions, | const std::set<std::string> &extensions, | ||||
| const std::map<std::string, int32_t> &class_indexing, | const std::map<std::string, int32_t> &class_indexing, | ||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| @@ -883,36 +1151,110 @@ ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool deco | |||||
| bool recursive = false; | bool recursive = false; | ||||
| // Create logical representation of ImageFolderDataset. | // Create logical representation of ImageFolderDataset. | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = | auto ds = | ||||
| std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler, recursive, extensions, class_indexing, cache); | |||||
| std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::set<std::string> &extensions, | |||||
| const std::map<std::string, int32_t> &class_indexing, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| // This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false. | |||||
| bool recursive = false; | |||||
| // Create logical representation of ImageFolderDataset. | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = | |||||
| std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, | |||||
| const std::shared_ptr<Sampler> &sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, | ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | const std::map<std::string, int32_t> &class_indexing, bool decode, | ||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler, class_indexing, decode, cache); | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, | MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, | ||||
| const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample, | |||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler, padded_sample, num_padded); | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files, | |||||
| const std::vector<std::string> &columns_list, const std::shared_ptr<Sampler> &sampler, | |||||
| nlohmann::json padded_sample, int64_t num_padded) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files, | |||||
| const std::vector<std::string> &columns_list, Sampler *sampler, | |||||
| nlohmann::json padded_sample, int64_t num_padded) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files, | MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files, | ||||
| const std::vector<std::string> &columns_list, | const std::vector<std::string> &columns_list, | ||||
| const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample, | |||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler, padded_sample, num_padded); | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| #endif | #endif | ||||
| MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, | MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, | ||||
| const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler, cache); | |||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| @@ -926,8 +1268,23 @@ TextFileDataset::TextFileDataset(const std::vector<std::string> &dataset_files, | |||||
| VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, | VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, | ||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | const std::map<std::string, int32_t> &class_indexing, bool decode, | ||||
| const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler, cache); | |||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | |||||
| auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | |||||
| } | |||||
| VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler.get().Parse(); | |||||
| auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache); | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| @@ -20,78 +20,68 @@ | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| /// Function to create a Distributed Sampler. | |||||
| std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle, | |||||
| int64_t num_samples, uint32_t seed, int64_t offset, | |||||
| bool even_dist) { | |||||
| auto sampler = | |||||
| std::make_shared<DistributedSamplerObj>(num_shards, shard_id, shuffle, num_samples, seed, offset, even_dist); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // DistributedSampler | |||||
| DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle, int64_t num_samples, | |||||
| uint32_t seed, int64_t offset, bool even_dist) | |||||
| : num_shards_(num_shards), | |||||
| shard_id_(shard_id), | |||||
| shuffle_(shuffle), | |||||
| num_samples_(num_samples), | |||||
| seed_(seed), | |||||
| offset_(offset), | |||||
| even_dist_(even_dist) {} | |||||
| std::shared_ptr<SamplerObj> DistributedSampler::Parse() { | |||||
| return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_, | |||||
| even_dist_); | |||||
| } | } | ||||
| /// Function to create a PK Sampler. | |||||
| std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) { | |||||
| auto sampler = std::make_shared<PKSamplerObj>(num_val, shuffle, num_samples); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // PKSampler | |||||
| PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) | |||||
| : num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {} | |||||
| std::shared_ptr<SamplerObj> PKSampler::Parse() { | |||||
| return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_); | |||||
| } | } | ||||
| /// Function to create a Random Sampler. | |||||
| std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement, int64_t num_samples) { | |||||
| auto sampler = std::make_shared<RandomSamplerObj>(replacement, num_samples); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // RandomSampler | |||||
| RandomSampler::RandomSampler(bool replacement, int64_t num_samples) | |||||
| : replacement_(replacement), num_samples_(num_samples) {} | |||||
| std::shared_ptr<SamplerObj> RandomSampler::Parse() { | |||||
| return std::make_shared<RandomSamplerObj>(replacement_, num_samples_); | |||||
| } | } | ||||
| /// Function to create a Sequential Sampler. | |||||
| std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index, int64_t num_samples) { | |||||
| auto sampler = std::make_shared<SequentialSamplerObj>(start_index, num_samples); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // SequentialSampler | |||||
| SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples) | |||||
| : start_index_(start_index), num_samples_(num_samples) {} | |||||
| std::shared_ptr<SamplerObj> SequentialSampler::Parse() { | |||||
| return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_); | |||||
| } | } | ||||
| /// Function to create a Subset Random Sampler. | |||||
| std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples) { | |||||
| auto sampler = std::make_shared<SubsetSamplerObj>(std::move(indices), num_samples); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // SubsetSampler | |||||
| SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples) | |||||
| : indices_(indices), num_samples_(num_samples) {} | |||||
| std::shared_ptr<SamplerObj> SubsetSampler::Parse() { | |||||
| return std::make_shared<SubsetSamplerObj>(indices_, num_samples_); | |||||
| } | } | ||||
| /// Function to create a Subset Random Sampler. | |||||
| std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples) { | |||||
| auto sampler = std::make_shared<SubsetRandomSamplerObj>(std::move(indices), num_samples); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // SubsetRandomSampler | |||||
| SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples) | |||||
| : SubsetSampler(indices, num_samples) {} | |||||
| std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() { | |||||
| return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_); | |||||
| } | } | ||||
| /// Function to create a Weighted Random Sampler. | |||||
| std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, | |||||
| bool replacement) { | |||||
| auto sampler = std::make_shared<WeightedRandomSamplerObj>(std::move(weights), num_samples, replacement); | |||||
| // Input validation | |||||
| if (sampler->ValidateParams().IsError()) { | |||||
| return nullptr; | |||||
| } | |||||
| return sampler; | |||||
| // WeightedRandomSampler | |||||
| WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement) | |||||
| : weights_(weights), num_samples_(num_samples), replacement_(replacement) {} | |||||
| std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() { | |||||
| return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_); | |||||
| } | } | ||||
| } // namespace dataset | } // namespace dataset | ||||
| @@ -138,7 +138,7 @@ Status ValidateDatasetShardParams(const std::string &dataset_name, int32_t num_s | |||||
| // Helper function to validate dataset sampler parameter | // Helper function to validate dataset sampler parameter | ||||
| Status ValidateDatasetSampler(const std::string &dataset_name, const std::shared_ptr<SamplerObj> &sampler) { | Status ValidateDatasetSampler(const std::string &dataset_name, const std::shared_ptr<SamplerObj> &sampler) { | ||||
| if (sampler == nullptr) { | |||||
| if (sampler == nullptr || sampler->ValidateParams().IsError()) { | |||||
| std::string err_msg = dataset_name + ": Sampler is not constructed correctly, sampler: nullptr"; | std::string err_msg = dataset_name + ": Sampler is not constructed correctly, sampler: nullptr"; | ||||
| MS_LOG(ERROR) << err_msg; | MS_LOG(ERROR) << err_msg; | ||||
| RETURN_STATUS_SYNTAX_ERROR(err_msg); | RETURN_STATUS_SYNTAX_ERROR(err_msg); | ||||
| @@ -191,17 +191,17 @@ std::shared_ptr<SamplerObj> SelectSampler(int64_t num_samples, bool shuffle, int | |||||
| if (shuffle) { | if (shuffle) { | ||||
| if (num_shards > 1) { | if (num_shards > 1) { | ||||
| // If shuffle enabled, sharding enabled, use distributed random sampler | // If shuffle enabled, sharding enabled, use distributed random sampler | ||||
| return DistributedSampler(num_shards, shard_id, shuffle, num_samples); | |||||
| return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse(); | |||||
| } | } | ||||
| // If shuffle enabled, sharding disabled, use random sampler | // If shuffle enabled, sharding disabled, use random sampler | ||||
| return RandomSampler(num_samples >= 0, num_samples); | |||||
| return RandomSampler(num_samples >= 0, num_samples).Parse(); | |||||
| } | } | ||||
| if (num_shards > 1) { | if (num_shards > 1) { | ||||
| // If shuffle disabled, sharding enabled, use distributed sequential sampler | // If shuffle disabled, sharding enabled, use distributed sequential sampler | ||||
| return DistributedSampler(num_shards, shard_id, shuffle, num_samples); | |||||
| return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse(); | |||||
| } | } | ||||
| // If shuffle disabled, sharding disabled, use sequential sampler | // If shuffle disabled, sharding disabled, use sequential sampler | ||||
| return SequentialSampler(0, num_samples); | |||||
| return SequentialSampler(0, num_samples).Parse(); | |||||
| } | } | ||||
| // Constructor to initialize the cache | // Constructor to initialize the cache | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -585,7 +585,13 @@ class AlbumDataset : public Dataset { | |||||
| public: | public: | ||||
| AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | ||||
| const std::vector<std::string> &column_names = {}, bool decode = false, | const std::vector<std::string> &column_names = {}, bool decode = false, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~AlbumDataset() = default; | ~AlbumDataset() = default; | ||||
| }; | }; | ||||
| @@ -597,20 +603,53 @@ class AlbumDataset : public Dataset { | |||||
| /// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns. | /// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns. | ||||
| /// (default = {}) | /// (default = {}) | ||||
| /// \param[in] decode the option to decode the images in dataset (default = false) | /// \param[in] decode the option to decode the images in dataset (default = false) | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | ||||
| const std::vector<std::string> &column_names = {}, bool decode = false, | const std::vector<std::string> &column_names = {}, bool decode = false, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create an AlbumDataset | |||||
| /// \notes The generated dataset is specified through setting a schema | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] data_schema Path to dataset schema file | |||||
| /// \param[in] column_names Column names used to specify columns to load | |||||
| /// \param[in] decode the option to decode the images in dataset | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create an AlbumDataset | |||||
| /// \notes The generated dataset is specified through setting a schema | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] data_schema Path to dataset schema file | |||||
| /// \param[in] column_names Column names used to specify columns to load | |||||
| /// \param[in] decode the option to decode the images in dataset | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | |||||
| const std::vector<std::string> &column_names, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| class CelebADataset : public Dataset { | class CelebADataset : public Dataset { | ||||
| public: | public: | ||||
| explicit CelebADataset(const std::string &dataset_dir, const std::string &usage = "all", | explicit CelebADataset(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), bool decode = false, | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| bool decode = false, const std::set<std::string> &extensions = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| bool decode = false, const std::set<std::string> &extensions = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit CelebADataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, bool decode = false, | |||||
| const std::set<std::string> &extensions = {}, | const std::set<std::string> &extensions = {}, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~CelebADataset() = default; | ~CelebADataset() = default; | ||||
| @@ -621,21 +660,54 @@ class CelebADataset : public Dataset { | |||||
| /// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. | /// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. | ||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset. | /// \param[in] dataset_dir Path to the root directory that contains the dataset. | ||||
| /// \param[in] usage One of "all", "train", "valid" or "test" (default = "all"). | /// \param[in] usage One of "all", "train", "valid" or "test" (default = "all"). | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] decode Decode the images after reading (default=false). | /// \param[in] decode Decode the images after reading (default=false). | ||||
| /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). | /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage = "all", | std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), bool decode = false, | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| bool decode = false, const std::set<std::string> &extensions = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a CelebADataset | |||||
| /// \notes The generated dataset has two columns ['image', 'attr']. | |||||
| /// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset. | |||||
| /// \param[in] usage One of "all", "train", "valid" or "test" | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] decode Decode the images after reading (default=false). | |||||
| /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| bool decode = false, const std::set<std::string> &extensions = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a CelebADataset | |||||
| /// \notes The generated dataset has two columns ['image', 'attr']. | |||||
| /// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset. | |||||
| /// \param[in] usage One of "all", "train", "valid" or "test" | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] decode Decode the images after reading (default=false). | |||||
| /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, bool decode = false, | |||||
| const std::set<std::string> &extensions = {}, | const std::set<std::string> &extensions = {}, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| class Cifar10Dataset : public Dataset { | class Cifar10Dataset : public Dataset { | ||||
| public: | public: | ||||
| explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage = "all", | explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~Cifar10Dataset() = default; | ~Cifar10Dataset() = default; | ||||
| }; | }; | ||||
| @@ -644,18 +716,43 @@ class Cifar10Dataset : public Dataset { | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | /// \notes The generated dataset has two columns ["image", "label"] | ||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | /// \param[in] dataset_dir Path to the root directory that contains the dataset | ||||
| /// \param[in] usage of CIFAR10, can be "train", "test" or "all" (default = "all"). | /// \param[in] usage of CIFAR10, can be "train", "test" or "all" (default = "all"). | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage = "all", | std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a Cifar10 Dataset | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] usage of CIFAR10, can be "train", "test" or "all" | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a Cifar10 Dataset | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] usage of CIFAR10, can be "train", "test" or "all" | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| class Cifar100Dataset : public Dataset { | class Cifar100Dataset : public Dataset { | ||||
| public: | public: | ||||
| explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage = "all", | explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~Cifar100Dataset() = default; | ~Cifar100Dataset() = default; | ||||
| }; | }; | ||||
| @@ -664,12 +761,32 @@ class Cifar100Dataset : public Dataset { | |||||
| /// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] | /// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] | ||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | /// \param[in] dataset_dir Path to the root directory that contains the dataset | ||||
| /// \param[in] usage of CIFAR100, can be "train", "test" or "all" (default = "all"). | /// \param[in] usage of CIFAR100, can be "train", "test" or "all" (default = "all"). | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage = "all", | std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a Cifar100 Dataset | |||||
| /// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] usage of CIFAR100, can be "train", "test" or "all". | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a Cifar100 Dataset | |||||
| /// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] usage of CIFAR100, can be "train", "test" or "all". | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| class CLUEDataset : public Dataset { | class CLUEDataset : public Dataset { | ||||
| @@ -707,7 +824,12 @@ std::shared_ptr<CLUEDataset> CLUE(const std::vector<std::string> &dataset_files, | |||||
| class CocoDataset : public Dataset { | class CocoDataset : public Dataset { | ||||
| public: | public: | ||||
| CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection", | CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection", | ||||
| const bool &decode = false, const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const bool &decode = false, const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, | |||||
| const bool &decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, | |||||
| const bool &decode, const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~CocoDataset() = default; | ~CocoDataset() = default; | ||||
| }; | }; | ||||
| @@ -725,13 +847,53 @@ class CocoDataset : public Dataset { | |||||
| /// \param[in] annotation_file Path to the annotation json | /// \param[in] annotation_file Path to the annotation json | ||||
| /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' | /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' | ||||
| /// \param[in] decode Decode the images after reading | /// \param[in] decode Decode the images after reading | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | ||||
| const std::string &task = "Detection", const bool &decode = false, | const std::string &task = "Detection", const bool &decode = false, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a CocoDataset | |||||
| /// \notes The generated dataset has multi-columns : | |||||
| /// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], | |||||
| /// ['iscrowd', dtype=uint32]]. | |||||
| /// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]]. | |||||
| /// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32], | |||||
| /// ['num_keypoints', dtype=uint32]]. | |||||
| /// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], | |||||
| /// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]]. | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] annotation_file Path to the annotation json | |||||
| /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' | |||||
| /// \param[in] decode Decode the images after reading | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | |||||
| const std::string &task, const bool &decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a CocoDataset | |||||
| /// \notes The generated dataset has multi-columns : | |||||
| /// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], | |||||
| /// ['iscrowd', dtype=uint32]]. | |||||
| /// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]]. | |||||
| /// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32], | |||||
| /// ['num_keypoints', dtype=uint32]]. | |||||
| /// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], | |||||
| /// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]]. | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] annotation_file Path to the annotation json | |||||
| /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' | |||||
| /// \param[in] decode Decode the images after reading | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | |||||
| const std::string &task, const bool &decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| class CSVDataset : public Dataset { | class CSVDataset : public Dataset { | ||||
| @@ -774,7 +936,16 @@ std::shared_ptr<CSVDataset> CSV(const std::vector<std::string> &dataset_files, c | |||||
| class ImageFolderDataset : public Dataset { | class ImageFolderDataset : public Dataset { | ||||
| public: | public: | ||||
| explicit ImageFolderDataset(const std::string &dataset_dir, bool decode = false, | explicit ImageFolderDataset(const std::string &dataset_dir, bool decode = false, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::set<std::string> &extensions = {}, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler, | |||||
| const std::set<std::string> &extensions = {}, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit ImageFolderDataset(const std::string &dataset_dir, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::set<std::string> &extensions = {}, | const std::set<std::string> &extensions = {}, | ||||
| const std::map<std::string, int32_t> &class_indexing = {}, | const std::map<std::string, int32_t> &class_indexing = {}, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| @@ -787,14 +958,46 @@ class ImageFolderDataset : public Dataset { | |||||
| /// The generated dataset has two columns ["image", "label"] | /// The generated dataset has two columns ["image", "label"] | ||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | /// \param[in] dataset_dir Path to the root directory that contains the dataset | ||||
| /// \param[in] decode A flag to decode in ImageFolder | /// \param[in] decode A flag to decode in ImageFolder | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] extensions File extensions to be read | /// \param[in] extensions File extensions to be read | ||||
| /// \param[in] class_indexing a class name to label map | /// \param[in] class_indexing a class name to label map | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current ImageFolderDataset | /// \return Shared pointer to the current ImageFolderDataset | ||||
| std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode = false, | |||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| std::shared_ptr<ImageFolderDataset> ImageFolder( | |||||
| const std::string &dataset_dir, bool decode = false, | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::set<std::string> &extensions = {}, const std::map<std::string, int32_t> &class_indexing = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create an ImageFolderDataset | |||||
| /// \notes A source dataset that reads images from a tree of directories | |||||
| /// All images within one folder have the same label | |||||
| /// The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] decode A flag to decode in ImageFolder | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] extensions File extensions to be read | |||||
| /// \param[in] class_indexing a class name to label map | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current ImageFolderDataset | |||||
| std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler, | |||||
| const std::set<std::string> &extensions = {}, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create an ImageFolderDataset | |||||
| /// \notes A source dataset that reads images from a tree of directories | |||||
| /// All images within one folder have the same label | |||||
| /// The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] decode A flag to decode in ImageFolder | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] extensions File extensions to be read | |||||
| /// \param[in] class_indexing a class name to label map | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current ImageFolderDataset | |||||
| std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::set<std::string> &extensions = {}, | const std::set<std::string> &extensions = {}, | ||||
| const std::map<std::string, int32_t> &class_indexing = {}, | const std::map<std::string, int32_t> &class_indexing = {}, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| @@ -802,7 +1005,14 @@ std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, | |||||
| class ManifestDataset : public Dataset { | class ManifestDataset : public Dataset { | ||||
| public: | public: | ||||
| explicit ManifestDataset(const std::string &dataset_file, const std::string &usage = "train", | explicit ManifestDataset(const std::string &dataset_file, const std::string &usage = "train", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit ManifestDataset(const std::string &dataset_file, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, | const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~ManifestDataset() = default; | ~ManifestDataset() = default; | ||||
| @@ -812,7 +1022,8 @@ class ManifestDataset : public Dataset { | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | /// \notes The generated dataset has two columns ["image", "label"] | ||||
| /// \param[in] dataset_file The dataset file to be read | /// \param[in] dataset_file The dataset file to be read | ||||
| /// \param[in] usage Need "train", "eval" or "inference" data (default="train") | /// \param[in] usage Need "train", "eval" or "inference" data (default="train") | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder | /// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder | ||||
| /// names will be sorted alphabetically and each class will be given a unique index starting from 0). | /// names will be sorted alphabetically and each class will be given a unique index starting from 0). | ||||
| @@ -820,19 +1031,56 @@ class ManifestDataset : public Dataset { | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current ManifestDataset | /// \return Shared pointer to the current ManifestDataset | ||||
| std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage = "train", | std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage = "train", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | |||||
| bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a ManifestDataset | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_file The dataset file to be read | |||||
| /// \param[in] usage Need "train", "eval" or "inference" data | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder | |||||
| /// names will be sorted alphabetically and each class will be given a unique index starting from 0). | |||||
| /// \param[in] decode Decode the images after reading (default=false). | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current ManifestDataset | |||||
| std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | |||||
| bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a ManifestDataset | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_file The dataset file to be read | |||||
| /// \param[in] usage Need "train", "eval" or "inference" data | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder | |||||
| /// names will be sorted alphabetically and each class will be given a unique index starting from 0). | |||||
| /// \param[in] decode Decode the images after reading (default=false). | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current ManifestDataset | |||||
| std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | const std::map<std::string, int32_t> &class_indexing = {}, | ||||
| bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr); | bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| class MindDataDataset : public Dataset { | class MindDataDataset : public Dataset { | ||||
| public: | public: | ||||
| explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list = {}, | explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list = {}, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | ||||
| explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | |||||
| explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample = nullptr, | |||||
| int64_t num_padded = 0); | |||||
| explicit MindDataDataset(const std::vector<std::string> &dataset_files, | explicit MindDataDataset(const std::vector<std::string> &dataset_files, | ||||
| const std::vector<std::string> &columns_list = {}, | const std::vector<std::string> &columns_list = {}, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | ||||
| explicit MindDataDataset(const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list, | |||||
| Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | |||||
| explicit MindDataDataset(const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list, | |||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample = nullptr, | |||||
| int64_t num_padded = 0); | |||||
| ~MindDataDataset() = default; | ~MindDataDataset() = default; | ||||
| }; | }; | ||||
| @@ -840,7 +1088,8 @@ class MindDataDataset : public Dataset { | |||||
| /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source | /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source | ||||
| /// in the same path will be found and loaded automatically. | /// in the same path will be found and loaded automatically. | ||||
| /// \param[in] columns_list List of columns to be read (default={}) | /// \param[in] columns_list List of columns to be read (default={}) | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), | ||||
| /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | ||||
| /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | ||||
| @@ -848,13 +1097,38 @@ class MindDataDataset : public Dataset { | |||||
| /// \return Shared pointer to the current MindDataDataset | /// \return Shared pointer to the current MindDataDataset | ||||
| std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, | std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, | ||||
| const std::vector<std::string> &columns_list = {}, | const std::vector<std::string> &columns_list = {}, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | |||||
| /// \brief Function to create a MindDataDataset | |||||
| /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source | |||||
| /// in the same path will be found and loaded automatically. | |||||
| /// \param[in] columns_list List of columns to be read | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | |||||
| /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | |||||
| /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | |||||
| /// \return Shared pointer to the current MindDataDataset | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| Sampler *sampler, nlohmann::json padded_sample = nullptr, | |||||
| int64_t num_padded = 0); | |||||
| /// \brief Function to create a MindDataDataset | |||||
| /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source | |||||
| /// in the same path will be found and loaded automatically. | |||||
| /// \param[in] columns_list List of columns to be read | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | |||||
| /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | |||||
| /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | |||||
| /// \return Shared pointer to the current MindDataDataset | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | ||||
| /// \brief Function to create a MindDataDataset | /// \brief Function to create a MindDataDataset | ||||
| /// \param[in] dataset_files List of dataset files to be read directly. | /// \param[in] dataset_files List of dataset files to be read directly. | ||||
| /// \param[in] columns_list List of columns to be read (default={}) | /// \param[in] columns_list List of columns to be read (default={}) | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), | ||||
| /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | ||||
| /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | ||||
| @@ -862,14 +1136,42 @@ std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, | |||||
| /// \return Shared pointer to the current MindDataDataset | /// \return Shared pointer to the current MindDataDataset | ||||
| std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | ||||
| const std::vector<std::string> &columns_list = {}, | const std::vector<std::string> &columns_list = {}, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | |||||
| /// \brief Function to create a MindDataDataset | |||||
| /// \param[in] dataset_files List of dataset files to be read directly. | |||||
| /// \param[in] columns_list List of columns to be read | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | |||||
| /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | |||||
| /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | |||||
| /// \return Shared pointer to the current MindDataDataset | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | |||||
| const std::vector<std::string> &columns_list, Sampler *sampler, | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | |||||
| /// \brief Function to create a MindDataDataset | |||||
| /// \param[in] dataset_files List of dataset files to be read directly. | |||||
| /// \param[in] columns_list List of columns to be read | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. | |||||
| /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. | |||||
| /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | |||||
| /// \return Shared pointer to the current MindDataDataset | |||||
| std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | |||||
| const std::vector<std::string> &columns_list, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); | ||||
| #endif | #endif | ||||
| class MnistDataset : public Dataset { | class MnistDataset : public Dataset { | ||||
| public: | public: | ||||
| explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all", | explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit MnistDataset(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~MnistDataset() = default; | ~MnistDataset() = default; | ||||
| }; | }; | ||||
| @@ -878,12 +1180,32 @@ class MnistDataset : public Dataset { | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | /// \notes The generated dataset has two columns ["image", "label"] | ||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | /// \param[in] dataset_dir Path to the root directory that contains the dataset | ||||
| /// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all"). | /// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all"). | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current MnistDataset | /// \return Shared pointer to the current MnistDataset | ||||
| std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all", | std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all", | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a MnistDataset | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] usage of MNIST, can be "train", "test" or "all" | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current MnistDataset | |||||
| std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a MnistDataset | |||||
| /// \notes The generated dataset has two columns ["image", "label"] | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] usage of MNIST, can be "train", "test" or "all" | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current MnistDataset | |||||
| std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| /// \brief Function to create a ConcatDataset | /// \brief Function to create a ConcatDataset | ||||
| @@ -909,8 +1231,6 @@ class RandomDataDataset : public Dataset { | |||||
| /// \param[in] total_rows Number of rows for the dataset to generate (default=0, number of rows is random) | /// \param[in] total_rows Number of rows for the dataset to generate (default=0, number of rows is random) | ||||
| /// \param[in] schema SchemaObj to set column type, data type and data shape | /// \param[in] schema SchemaObj to set column type, data type and data shape | ||||
| /// \param[in] columns_list List of columns to be read (default={}, read all columns) | /// \param[in] columns_list List of columns to be read (default={}, read all columns) | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| template <typename T = std::shared_ptr<SchemaObj>> | template <typename T = std::shared_ptr<SchemaObj>> | ||||
| @@ -1027,7 +1347,14 @@ class VOCDataset : public Dataset { | |||||
| public: | public: | ||||
| explicit VOCDataset(const std::string &dataset_dir, const std::string &task = "Segmentation", | explicit VOCDataset(const std::string &dataset_dir, const std::string &task = "Segmentation", | ||||
| const std::string &usage = "train", const std::map<std::string, int32_t> &class_indexing = {}, | const std::string &usage = "train", const std::map<std::string, int32_t> &class_indexing = {}, | ||||
| bool decode = false, const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| bool decode = false, const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| ~VOCDataset() = default; | ~VOCDataset() = default; | ||||
| }; | }; | ||||
| @@ -1042,14 +1369,48 @@ class VOCDataset : public Dataset { | |||||
| /// \param[in] usage The type of data list text file to be read (default = "train"). | /// \param[in] usage The type of data list text file to be read (default = "train"). | ||||
| /// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task | /// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task | ||||
| /// \param[in] decode Decode the images after reading | /// \param[in] decode Decode the images after reading | ||||
| /// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, | |||||
| /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not | |||||
| /// given, | |||||
| /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task = "Segmentation", | std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task = "Segmentation", | ||||
| const std::string &usage = "train", | const std::string &usage = "train", | ||||
| const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, | const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false, | ||||
| const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), | |||||
| const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a VOCDataset | |||||
| /// \notes The generated dataset has multi-columns : | |||||
| /// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32], | |||||
| /// ['difficult', dtype=uint32], ['truncate', dtype=uint32]]. | |||||
| /// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]]. | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection" | |||||
| /// \param[in] usage The type of data list text file to be read. | |||||
| /// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task | |||||
| /// \param[in] decode Decode the images after reading | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | |||||
| /// \brief Function to create a VOCDataset | |||||
| /// \notes The generated dataset has multi-columns : | |||||
| /// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32], | |||||
| /// ['difficult', dtype=uint32], ['truncate', dtype=uint32]]. | |||||
| /// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]]. | |||||
| /// \param[in] dataset_dir Path to the root directory that contains the dataset | |||||
| /// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection" | |||||
| /// \param[in] usage The type of data list text file to be read. | |||||
| /// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task | |||||
| /// \param[in] decode Decode the images after reading | |||||
| /// \param[in] sampler Sampler object used to choose samples from the dataset. | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | |||||
| /// \return Shared pointer to the current Dataset | |||||
| std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, | |||||
| const std::map<std::string, int32_t> &class_indexing, bool decode, | |||||
| const std::reference_wrapper<Sampler> sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr); | const std::shared_ptr<DatasetCache> &cache = nullptr); | ||||
| /// \brief Function the create a cache to be attached to a dataset | /// \brief Function the create a cache to be attached to a dataset | ||||
| @@ -26,76 +26,225 @@ | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| class DistributedSamplerObj; | |||||
| class PKSamplerObj; | |||||
| class PreBuiltSamplerObj; | |||||
| class RandomSamplerObj; | |||||
| class SequentialSamplerObj; | |||||
| class SubsetSamplerObj; | |||||
| class SubsetRandomSamplerObj; | |||||
| class WeightedRandomSamplerObj; | |||||
| /// Function to create a Distributed Sampler. | |||||
| /// \notes A Sampler that access a shard of the dataset. | |||||
| /// \param[in] num_shards - Number of shards to divide the dataset into. | |||||
| /// \param[in] shard_id - Shard ID of the current shard within num_shards. | |||||
| /// \param[in] shuffle - If true, the indices are shuffled. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \param[in] seed - The seed in use when shuffle is true. | |||||
| /// \param[in] offset - The starting position where access to elements in the dataset begins. | |||||
| /// \param[in] even_dist - If true, each shard would return the same number of rows (default to true). | |||||
| /// If false the total rows returned by all the shards would not have overlap. | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, | |||||
| int64_t num_samples = 0, uint32_t seed = 1, | |||||
| int64_t offset = -1, bool even_dist = true); | |||||
| /// Function to create a PK Sampler. | |||||
| // Abstract class to represent a sampler in the data pipeline. | |||||
| /// \class Sampler samplers.h | |||||
| /// \brief An abstract base class to represent a sampler in the data pipeline. | |||||
| class Sampler : std::enable_shared_from_this<Sampler> { | |||||
| friend class AlbumDataset; | |||||
| friend class CelebADataset; | |||||
| friend class Cifar10Dataset; | |||||
| friend class Cifar100Dataset; | |||||
| friend class CLUEDataset; | |||||
| friend class CocoDataset; | |||||
| friend class CSVDataset; | |||||
| friend class ImageFolderDataset; | |||||
| friend class ManifestDataset; | |||||
| friend class MindDataDataset; | |||||
| friend class MnistDataset; | |||||
| friend class RandomDataDataset; | |||||
| friend class TextFileDataset; | |||||
| friend class TFRecordDataset; | |||||
| friend class VOCDataset; | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| Sampler() {} | |||||
| /// \brief Destructor | |||||
| ~Sampler() = default; | |||||
| /// \brief A virtual function to add a child sampler. | |||||
| /// \param[in] child The child sampler to be added as a children of this sampler. | |||||
| virtual void AddChild(std::shared_ptr<Sampler> child) { children_.push_back(child); } | |||||
| protected: | |||||
| /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. | |||||
| /// \return shared pointer to the newly created TensorOperation. | |||||
| virtual std::shared_ptr<SamplerObj> Parse() = 0; | |||||
| std::vector<std::shared_ptr<Sampler>> children_; | |||||
| }; | |||||
| /// \brief A class to represent a Distributed Sampler in the data pipeline. | |||||
| /// \notes A Sampler that accesses a shard of the dataset. | |||||
| class DistributedSampler : public Sampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] num_shards - Number of shards to divide the dataset into. | |||||
| /// \param[in] shard_id - Shard ID of the current shard within num_shards. | |||||
| /// \param[in] shuffle - If true, the indices are shuffled. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \param[in] seed - The seed in use when shuffle is true. | |||||
| /// \param[in] offset - The starting position where access to elements in the dataset begins. | |||||
| /// \param[in] even_dist - If true, each shard would return the same number of rows (default to true). | |||||
| /// If false the total rows returned by all the shards would not have overlap. | |||||
| explicit DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, int64_t num_samples = 0, | |||||
| uint32_t seed = 1, int64_t offset = -1, bool even_dist = true); | |||||
| /// \brief Destructor. | |||||
| ~DistributedSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| private: | |||||
| int64_t num_shards_; | |||||
| int64_t shard_id_; | |||||
| bool shuffle_; | |||||
| int64_t num_samples_; | |||||
| uint32_t seed_; | |||||
| int64_t offset_; | |||||
| bool even_dist_; | |||||
| }; | |||||
| /// \brief A class to represent a PK Sampler in the data pipeline. | |||||
| /// \notes Samples K elements for each P class in the dataset. | /// \notes Samples K elements for each P class in the dataset. | ||||
| /// This will sample all classes. | /// This will sample all classes. | ||||
| /// \param[in] num_val - Number of elements to sample for each class. | |||||
| /// \param[in] shuffle - If true, the class IDs are shuffled. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0); | |||||
| class PKSampler : public Sampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] num_val - Number of elements to sample for each class. | |||||
| /// \param[in] shuffle - If true, the class IDs are shuffled. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| explicit PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0); | |||||
| /// \brief Destructor. | |||||
| ~PKSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| /// Function to create a Random Sampler. | |||||
| private: | |||||
| int64_t num_val_; | |||||
| bool shuffle_; | |||||
| int64_t num_samples_; | |||||
| }; | |||||
| /// \brief A class to represent a Random Sampler in the data pipeline. | |||||
| /// \notes Samples the elements randomly. | /// \notes Samples the elements randomly. | ||||
| /// \param[in] replacement - If true, put the sample ID back for the next draw. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement = false, int64_t num_samples = 0); | |||||
| class RandomSampler : public Sampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] replacement - If true, put the sample ID back for the next draw. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| explicit RandomSampler(bool replacement = false, int64_t num_samples = 0); | |||||
| /// \brief Destructor. | |||||
| ~RandomSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| /// Function to create a Sequential Sampler. | |||||
| private: | |||||
| bool replacement_; | |||||
| int64_t num_samples_; | |||||
| }; | |||||
| /// \brief A class to represent a Sequential Sampler in the data pipeline. | |||||
| /// \notes Samples the dataset elements sequentially, same as not having a sampler. | /// \notes Samples the dataset elements sequentially, same as not having a sampler. | ||||
| /// \param[in] start_index - Index to start sampling at (default to start at first id). | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0); | |||||
| class SequentialSampler : public Sampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] start_index - Index to start sampling at (default to start at first id). | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| explicit SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0); | |||||
| /// \brief Destructor. | |||||
| ~SequentialSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| /// Function to create a Subset Sampler. | |||||
| private: | |||||
| int64_t start_index_; | |||||
| int64_t num_samples_; | |||||
| }; | |||||
| /// \brief A class to represent a Subset Sampler in the data pipeline. | |||||
| /// \notes Samples the elements from a sequence of indices. | /// \notes Samples the elements from a sequence of indices. | ||||
| /// \param[in] indices - A vector sequence of indices. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0); | |||||
| class SubsetSampler : public Sampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] indices - A vector sequence of indices. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| explicit SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0); | |||||
| /// \brief Destructor. | |||||
| ~SubsetSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| /// Function to create a Subset Random Sampler. | |||||
| std::vector<int64_t> indices_; | |||||
| int64_t num_samples_; | |||||
| }; | |||||
| /// \brief A class to represent a Subset Random Sampler in the data pipeline. | |||||
| /// \notes Samples the elements randomly from a sequence of indices. | /// \notes Samples the elements randomly from a sequence of indices. | ||||
| /// \param[in] indices - A vector sequence of indices. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0); | |||||
| class SubsetRandomSampler : public SubsetSampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] indices - A vector sequence of indices. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| explicit SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0); | |||||
| /// \brief Destructor. | |||||
| ~SubsetRandomSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| }; | |||||
| /// Function to create a Weighted Random Sampler. | |||||
| /// \brief A class to represent a Weighted Random Sampler in the data pipeline. | |||||
| /// \notes Samples the elements from [0, len(weights) - 1] randomly with the given | /// \notes Samples the elements from [0, len(weights) - 1] randomly with the given | ||||
| /// weights (probabilities). | /// weights (probabilities). | ||||
| /// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \param[in] replacement - If true, put the sample ID back for the next draw. | |||||
| /// \return Shared pointer to the current Sampler. | |||||
| std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0, | |||||
| bool replacement = true); | |||||
| class WeightedRandomSampler : public Sampler { | |||||
| friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t); | |||||
| public: | |||||
| /// \brief Constructor | |||||
| /// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1. | |||||
| /// \param[in] num_samples - The number of samples to draw (default to all elements). | |||||
| /// \param[in] replacement - If true, put the sample ID back for the next draw. | |||||
| explicit WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0, bool replacement = true); | |||||
| /// \brief Destructor. | |||||
| ~WeightedRandomSampler() = default; | |||||
| protected: | |||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | |||||
| /// \return shared pointer to the newly created SamplerObj. | |||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| private: | |||||
| std::vector<double> weights_; | |||||
| int64_t num_samples_; | |||||
| bool replacement_; | |||||
| }; | |||||
| } // namespace dataset | } // namespace dataset | ||||
| } // namespace mindspore | } // namespace mindspore | ||||
| @@ -57,7 +57,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCApiNestedCache) { | |||||
| // Create an ImageFolder Dataset, this folder_path only has 2 images in it | // Create an ImageFolder Dataset, this folder_path only has 2 images in it | ||||
| std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -85,7 +85,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheImageFolderCApi) { | |||||
| // Create an ImageFolder Dataset, this folder_path only has 2 images in it | // Create an ImageFolder Dataset, this folder_path only has 2 images in it | ||||
| std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -128,7 +128,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCocoCApi) { | |||||
| std::string folder_path = datasets_root_path_ + "/testCOCO/train/"; | std::string folder_path = datasets_root_path_ + "/testCOCO/train/"; | ||||
| std::string annotation_file_path = datasets_root_path_ + "/testCOCO/annotations/train.json"; | std::string annotation_file_path = datasets_root_path_ + "/testCOCO/annotations/train.json"; | ||||
| std::shared_ptr<Dataset> ds = | std::shared_ptr<Dataset> ds = | ||||
| Coco(folder_path, annotation_file_path, "Detection", false, RandomSampler(), some_cache); | |||||
| Coco(folder_path, annotation_file_path, "Detection", false, std::make_shared<RandomSampler>(), some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -169,7 +169,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheMnistCApi) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10), some_cache); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -210,7 +210,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) { | |||||
| // Create a CelebA Dataset, this folder_path has 4 records in it | // Create a CelebA Dataset, this folder_path has 4 records in it | ||||
| std::string folder_path = datasets_root_path_ + "/testCelebAData/"; | std::string folder_path = datasets_root_path_ + "/testCelebAData/"; | ||||
| std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", RandomSampler(false, 10), false, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", std::make_shared<RandomSampler>(false, 10), false, {}, some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -251,7 +251,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheManifestCApi) { | |||||
| // Create a Manifest Dataset, this file_path has 2 records in it | // Create a Manifest Dataset, this file_path has 2 records in it | ||||
| std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; | std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; | ||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), {}, false, some_cache); | |||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), {}, false, some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -292,7 +292,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar10CApi) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10), some_cache); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -333,7 +333,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar100CApi) { | |||||
| // Create a Cifar100 Dataset | // Create a Cifar100 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10), some_cache); | |||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -374,7 +374,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) { | |||||
| // Create a VOC Dataset, this folder_path has 9 records in it | // Create a VOC Dataset, this folder_path has 9 records in it | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012/"; | std::string folder_path = datasets_root_path_ + "/testVOC2012/"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, RandomSampler(), some_cache); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<RandomSampler>(), some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -417,7 +417,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheAlbumCApi) { | |||||
| std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; | std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; | ||||
| std::vector<std::string> column_names = {"image", "label", "id"}; | std::vector<std::string> column_names = {"image", "label", "id"}; | ||||
| // Create a Album Dataset, 7 records in it | // Create a Album Dataset, 7 records in it | ||||
| std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, false, RandomSampler(), some_cache); | |||||
| std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, false, std::make_shared<RandomSampler>(), some_cache); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -770,9 +770,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) { | |||||
| // Create an ImageFolder Dataset, this folder_path only has 2 images in it | // Create an ImageFolder Dataset, this folder_path only has 2 images in it | ||||
| std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create and launch the Execution Tree for ds1 | // Create and launch the Execution Tree for ds1 | ||||
| @@ -824,9 +824,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) { | |||||
| std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | ||||
| // The first pipeline is ImageFolder with RandomSampler, the second pipeline is ImageFolder with SequentialSampler | // The first pipeline is ImageFolder with RandomSampler, the second pipeline is ImageFolder with SequentialSampler | ||||
| // Since sampler does not influence the data in the source, these two pipelines can share a common cache. | // Since sampler does not influence the data in the source, these two pipelines can share a common cache. | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, SequentialSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create and launch the Execution Tree for ds1 | // Create and launch the Execution Tree for ds1 | ||||
| @@ -874,9 +874,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) { | |||||
| // Create an ImageFolder Dataset, this folder_path only has 2 images in it | // Create an ImageFolder Dataset, this folder_path only has 2 images in it | ||||
| std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create and launch the Execution Tree for ds1 | // Create and launch the Execution Tree for ds1 | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) { | |||||
| std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; | std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; | ||||
| std::vector<std::string> column_names = {"image", "label", "id"}; | std::vector<std::string> column_names = {"image", "label", "id"}; | ||||
| // Create a Album Dataset | // Create a Album Dataset | ||||
| std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1)); | |||||
| std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -206,7 +206,7 @@ TEST_F(MindDataTestPipeline, TestAlbumError) { | |||||
| std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; | std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; | ||||
| std::vector<std::string> column_names = {"image", "label", "id"}; | std::vector<std::string> column_names = {"image", "label", "id"}; | ||||
| // Create an Album Dataset | // Create an Album Dataset | ||||
| std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1)); | |||||
| std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -64,8 +64,8 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) { | |||||
| // Create two Cifar10 Dataset | // Create two Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds1 = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds1 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| @@ -168,7 +168,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Dataset) { | |||||
| // Create a Cifar100 Dataset | // Create a Cifar100 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -203,7 +203,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Getters) { | |||||
| // Create a Cifar100 Dataset | // Create a Cifar100 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::vector<std::string> column_names = {"image", "coarse_label", "fine_label"}; | std::vector<std::string> column_names = {"image", "coarse_label", "fine_label"}; | ||||
| @@ -230,7 +230,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetFail) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100DatasetFail."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100DatasetFail."; | ||||
| // Create a Cifar100 Dataset | // Create a Cifar100 Dataset | ||||
| std::shared_ptr<Dataset> ds = Cifar100("", "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar100("", "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -243,7 +243,7 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetFail) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail."; | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::shared_ptr<Dataset> ds = Cifar10("", "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10("", "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetWithWrongSamplerFail) { | |||||
| // Create a Cifar100 Dataset | // Create a Cifar100 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, -10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, -10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -138,7 +138,7 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) { | |||||
| std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | ||||
| std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; | std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; | ||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6)); | |||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, std::make_shared<SequentialSampler>(0, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -220,7 +220,7 @@ TEST_F(MindDataTestPipeline, TestCocoKeypoint) { | |||||
| std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | ||||
| std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json"; | std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json"; | ||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, SequentialSampler(0, 2)); | |||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, std::make_shared<SequentialSampler>(0, 2)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -275,7 +275,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) { | |||||
| std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | ||||
| std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; | std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; | ||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2)); | |||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -333,7 +333,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanopticGetClassIndex) { | |||||
| std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | ||||
| std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; | std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; | ||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2)); | |||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); | std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); | ||||
| @@ -355,7 +355,7 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) { | |||||
| std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | std::string folder_path = datasets_root_path_ + "/testCOCO/train"; | ||||
| std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; | std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; | ||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6)); | |||||
| std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, std::make_shared<SequentialSampler>(0, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -1,6 +1,6 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -29,7 +29,7 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn."; | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Rename operation on ds | // Create a Rename operation on ds | ||||
| @@ -65,7 +65,7 @@ TEST_F(MindDataTestPipeline, TestIteratorOneColumn) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; | ||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 4)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestIteratorReOrder) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder."; | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(false, 4)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(false, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Take operation on ds | // Create a Take operation on ds | ||||
| @@ -144,7 +144,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; | ||||
| // Create a VOC Dataset | // Create a VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -187,7 +187,7 @@ TEST_F(MindDataTestPipeline, TestIteratorWrongColumn) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; | ||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 4)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Pass wrong column name | // Pass wrong column name | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -147,7 +147,7 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) { | |||||
| std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; | std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; | ||||
| // Create a Manifest Dataset | // Create a Manifest Dataset | ||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), {}, true); | |||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), {}, true); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -218,7 +218,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) { | |||||
| std::vector<int64_t> expected_label = {111, 222}; | std::vector<int64_t> expected_label = {111, 222}; | ||||
| // Create a Manifest Dataset | // Create a Manifest Dataset | ||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), map, true); | |||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), map, true); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); | std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); | ||||
| @@ -261,7 +261,7 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) { | |||||
| std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; | std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; | ||||
| // Create a Manifest Dataset | // Create a Manifest Dataset | ||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", SequentialSampler(0, 1), {}, true); | |||||
| std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<SequentialSampler>(0, 1), {}, true); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -182,7 +182,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess5) { | |||||
| // Pass one mindrecord shard file to parse dataset info, and search for other mindrecord files with same dataset info, | // Pass one mindrecord shard file to parse dataset info, and search for other mindrecord files with same dataset info, | ||||
| // thus all records in imagenet.mindrecord0 ~ imagenet.mindrecord3 will be read | // thus all records in imagenet.mindrecord0 ~ imagenet.mindrecord3 will be read | ||||
| std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::shared_ptr<Dataset> ds = MindData(file_path1, {}, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = MindData(file_path1, {}, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -223,26 +223,26 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) { | |||||
| std::vector<std::string> file_list = {file_path1}; | std::vector<std::string> file_list = {file_path1}; | ||||
| // Check sequential sampler, output number is 5 | // Check sequential sampler, output number is 5 | ||||
| std::shared_ptr<Dataset> ds1 = MindData(file_list, {}, SequentialSampler(0, 10)); | |||||
| std::shared_ptr<Dataset> ds1 = MindData(file_list, {}, std::make_shared<SequentialSampler>(0, 10)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Check random sampler, output number is 5, same rows with file | // Check random sampler, output number is 5, same rows with file | ||||
| std::shared_ptr<Dataset> ds2 = MindData(file_list, {}, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = MindData(file_list, {}, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Check pk sampler, output number is 2, get 2 samples with label 0 | // Check pk sampler, output number is 2, get 2 samples with label 0 | ||||
| std::shared_ptr<Dataset> ds3 = MindData(file_list, {}, PKSampler(2, false, 10)); | |||||
| std::shared_ptr<Dataset> ds3 = MindData(file_list, {}, std::make_shared<PKSampler>(2, false, 10)); | |||||
| EXPECT_NE(ds3, nullptr); | EXPECT_NE(ds3, nullptr); | ||||
| // Check distributed sampler, output number is 3, get 3 samples in shard 0 | // Check distributed sampler, output number is 3, get 3 samples in shard 0 | ||||
| std::shared_ptr<Dataset> ds4 = MindData(file_list, {}, DistributedSampler(2, 0, false, 10)); | |||||
| std::shared_ptr<Dataset> ds4 = MindData(file_list, {}, std::make_shared<DistributedSampler>(2, 0, false, 10)); | |||||
| EXPECT_NE(ds4, nullptr); | EXPECT_NE(ds4, nullptr); | ||||
| // Check distributed sampler get 3 samples with indice 0, 1 ,2 | // Check distributed sampler get 3 samples with indice 0, 1 ,2 | ||||
| std::shared_ptr<Dataset> ds5 = MindData(file_list, {}, SubsetRandomSampler({0, 1, 2}, 10)); | |||||
| std::shared_ptr<Dataset> ds5 = MindData(file_list, {}, new SubsetRandomSampler({0, 1, 2}, 10)); | |||||
| EXPECT_NE(ds5, nullptr); | EXPECT_NE(ds5, nullptr); | ||||
| std::shared_ptr<Dataset> ds6 = MindData(file_list, {}, SubsetSampler({1, 2}, 10)); | |||||
| std::shared_ptr<Dataset> ds6 = MindData(file_list, {}, new SubsetSampler({1, 2}, 10)); | |||||
| EXPECT_NE(ds5, nullptr); | EXPECT_NE(ds5, nullptr); | ||||
| std::vector<std::shared_ptr<Dataset>> ds = {ds1, ds2, ds3, ds4, ds5, ds6}; | std::vector<std::shared_ptr<Dataset>> ds = {ds1, ds2, ds3, ds4, ds5, ds6}; | ||||
| @@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) { | |||||
| // Pass a list of mindrecord file name, files in list will be read directly but not search for related files | // Pass a list of mindrecord file name, files in list will be read directly but not search for related files | ||||
| std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::vector<std::string> file_list = {file_path1}; | std::vector<std::string> file_list = {file_path1}; | ||||
| std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); | |||||
| std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Skip operation on ds, skip original data in mindrecord and get padded samples | // Create a Skip operation on ds, skip original data in mindrecord and get padded samples | ||||
| @@ -332,7 +332,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) { | |||||
| // Pass a list of mindrecord file name, files in list will be read directly but not search for related files | // Pass a list of mindrecord file name, files in list will be read directly but not search for related files | ||||
| std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::vector<std::string> file_list = {file_path1}; | std::vector<std::string> file_list = {file_path1}; | ||||
| std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); | |||||
| std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::vector<DataType> types = ds->GetOutputTypes(); | std::vector<DataType> types = ds->GetOutputTypes(); | ||||
| @@ -400,12 +400,12 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) { | |||||
| // Pass a list of mindrecord file name, files in list will be read directly but not search for related files | // Pass a list of mindrecord file name, files in list will be read directly but not search for related files | ||||
| std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::vector<std::string> file_list = {file_path1}; | std::vector<std::string> file_list = {file_path1}; | ||||
| std::shared_ptr<Dataset> ds1 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); | |||||
| std::shared_ptr<Dataset> ds1 = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| ds1 = ds1->Skip(5); | ds1 = ds1->Skip(5); | ||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); | |||||
| std::shared_ptr<Dataset> ds2 = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| ds2 = ds2->Skip(5); | ds2 = ds2->Skip(5); | ||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| @@ -534,7 +534,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail3) { | |||||
| // Create a MindData Dataset with unsupported sampler | // Create a MindData Dataset with unsupported sampler | ||||
| std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, WeightedRandomSampler({1, 1, 1, 1})); | |||||
| std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, new WeightedRandomSampler({1, 1, 1, 1})); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -558,7 +558,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { | |||||
| // Create a MindData Dataset | // Create a MindData Dataset | ||||
| std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, RandomSampler(), nullptr, 2); | |||||
| std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, std::make_shared<RandomSampler>(), nullptr, 2); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { | |||||
| // Create a MindData Dataset | // Create a MindData Dataset | ||||
| std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, RandomSampler(), pad, -2); | |||||
| std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, std::make_shared<RandomSampler>(), pad, -2); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -583,7 +583,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { | |||||
| // Create a MindData Dataset | // Create a MindData Dataset | ||||
| std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, RandomSampler(), pad, 1); | |||||
| std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, std::make_shared<RandomSampler>(), pad, 1); | |||||
| EXPECT_NE(ds3, nullptr); | EXPECT_NE(ds3, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -598,7 +598,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { | |||||
| // Create a MindData Dataset | // Create a MindData Dataset | ||||
| std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; | ||||
| std::shared_ptr<Dataset> ds4 = MindData(file_path4, {"file_name", "label"}, RandomSampler(), pad2, 1); | |||||
| std::shared_ptr<Dataset> ds4 = MindData(file_path4, {"file_name", "label"}, std::make_shared<RandomSampler>(), pad2, 1); | |||||
| EXPECT_NE(ds4, nullptr); | EXPECT_NE(ds4, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -118,7 +118,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess1) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -154,7 +154,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -192,7 +192,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail1) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -212,7 +212,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail2) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -232,7 +232,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail3) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -252,7 +252,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail4) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -272,7 +272,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail5) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -292,7 +292,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail6) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, -2, 3}); | ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, -2, 3}); | ||||
| @@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail7) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a BucketBatchByLength operation on ds | // Create a BucketBatchByLength operation on ds | ||||
| @@ -331,9 +331,9 @@ TEST_F(MindDataTestPipeline, TestConcatFail1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Rename operation on ds | // Create a Rename operation on ds | ||||
| @@ -362,7 +362,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Concat operation on the ds | // Create a Concat operation on the ds | ||||
| @@ -382,7 +382,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail3) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Concat operation on the ds | // Create a Concat operation on the ds | ||||
| @@ -402,7 +402,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail4) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Concat operation on the ds | // Create a Concat operation on the ds | ||||
| @@ -422,10 +422,10 @@ TEST_F(MindDataTestPipeline, TestConcatFail5) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -451,13 +451,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| folder_path = datasets_root_path_ + "/testCifar10Data/"; | folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -502,13 +502,13 @@ TEST_F(MindDataTestPipeline, TestConcatGetDatasetSize) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| folder_path = datasets_root_path_ + "/testCifar10Data/"; | folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -531,13 +531,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| // Column names: {"image", "label"} | // Column names: {"image", "label"} | ||||
| folder_path = datasets_root_path_ + "/testCifar10Data/"; | folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -755,7 +755,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -796,7 +796,7 @@ TEST_F(MindDataTestPipeline, TestPipelineGetDatasetSize) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -818,7 +818,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<DistributedSampler>(4, 0, false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // num_per_shard is equal to 44/4 = 11 which is more than num_samples = 10, so the output is 10 | // num_per_shard is equal to 44/4 = 11 which is more than num_samples = 10, so the output is 10 | ||||
| @@ -849,7 +849,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 15)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<DistributedSampler>(4, 0, false, 15)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // num_per_shard is equal to 44/4 = 11 which is less than num_samples = 15, so the output is 11 | // num_per_shard is equal to 44/4 = 11 which is less than num_samples = 15, so the output is 11 | ||||
| @@ -879,7 +879,7 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -933,7 +933,7 @@ TEST_F(MindDataTestPipeline, TestProjectDuplicateColumnFail) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -962,7 +962,7 @@ TEST_F(MindDataTestPipeline, TestMapDuplicateColumnFail) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1002,7 +1002,7 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1054,7 +1054,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1078,7 +1078,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Rename operation on ds | // Create a Rename operation on ds | ||||
| @@ -1097,7 +1097,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail3) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Rename operation on ds | // Create a Rename operation on ds | ||||
| @@ -1124,7 +1124,7 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1174,7 +1174,7 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1217,7 +1217,7 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1256,7 +1256,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1276,7 +1276,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1295,7 +1295,7 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Shuffle operation on ds | // Create a Shuffle operation on ds | ||||
| @@ -1341,7 +1341,7 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Skip operation on ds | // Create a Skip operation on ds | ||||
| @@ -1379,7 +1379,7 @@ TEST_F(MindDataTestPipeline, TestSkipTakeRepeat) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 6)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 6)); | |||||
| // Create a Skip operation on ds | // Create a Skip operation on ds | ||||
| int32_t count = 0; | int32_t count = 0; | ||||
| @@ -1424,7 +1424,7 @@ TEST_F(MindDataTestPipeline, TestSkipGetDatasetSize) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Skip operation on ds | // Create a Skip operation on ds | ||||
| @@ -1440,7 +1440,7 @@ TEST_F(MindDataTestPipeline, TestSkipDatasetError1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Skip operation on ds with invalid count input | // Create a Skip operation on ds with invalid count input | ||||
| @@ -1459,7 +1459,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Take operation on ds, default count = -1 | // Create a Take operation on ds, default count = -1 | ||||
| @@ -1496,7 +1496,7 @@ TEST_F(MindDataTestPipeline, TestTakeGetDatasetSize) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Take operation on ds, default count = -1 | // Create a Take operation on ds, default count = -1 | ||||
| @@ -1511,7 +1511,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetError1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Take operation on ds with invalid count input | // Create a Take operation on ds with invalid count input | ||||
| @@ -1540,7 +1540,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 8)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 8)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Take operation on ds | // Create a Take operation on ds | ||||
| @@ -1577,7 +1577,7 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1631,11 +1631,11 @@ TEST_F(MindDataTestPipeline, TestZipFail) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Create a Zip operation on the datasets | // Create a Zip operation on the datasets | ||||
| @@ -1659,7 +1659,7 @@ TEST_F(MindDataTestPipeline, TestZipFail2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Zip operation on the datasets | // Create a Zip operation on the datasets | ||||
| @@ -1679,7 +1679,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -1688,7 +1688,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) | // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) | ||||
| @@ -1696,7 +1696,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| folder_path = datasets_root_path_ + "/testCifar10Data/"; | folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -1749,7 +1749,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -1758,7 +1758,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) { | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 3)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) | // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) | ||||
| @@ -1766,7 +1766,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) { | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| folder_path = datasets_root_path_ + "/testCifar10Data/"; | folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Project operation on ds | // Create a Project operation on ds | ||||
| @@ -1787,9 +1787,9 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 9)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 9)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names) | // Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names) | ||||
| @@ -1840,7 +1840,7 @@ TEST_F(MindDataTestPipeline, TestNumWorkersValidate) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 1)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 1)); | |||||
| // ds needs to be non nullptr otherwise, the subsequent logic will core dump | // ds needs to be non nullptr otherwise, the subsequent logic will core dump | ||||
| ASSERT_NE(ds, nullptr); | ASSERT_NE(ds, nullptr); | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) { | |||||
| // Stage 1: load original dataset | // Stage 1: load original dataset | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(0, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(0, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -70,7 +70,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) { | |||||
| // Stage 3: Load dataset from file output by stage 2 | // Stage 3: Load dataset from file output by stage 2 | ||||
| // Create a MindData Dataset | // Create a MindData Dataset | ||||
| std::shared_ptr<Dataset> ds_minddata = MindData(temp_file, {}, SequentialSampler(0, 10)); | |||||
| std::shared_ptr<Dataset> ds_minddata = MindData(temp_file, {}, std::make_shared<SequentialSampler>(0, 10)); | |||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| // uint32 will be casted to int64 implicitly in mindrecord file, so we have to cast it back to uint32 | // uint32 will be casted to int64 implicitly in mindrecord file, so we have to cast it back to uint32 | ||||
| @@ -119,7 +119,7 @@ TEST_F(MindDataTestPipeline, TestSaveFail) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(0, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(0, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // fail with invalid dataset_path | // fail with invalid dataset_path | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -35,7 +35,7 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) { | |||||
| class_index["cat"] = 1; | class_index["cat"] = 1; | ||||
| class_index["train"] = 9; | class_index["train"] = 9; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetClassIndex) { | |||||
| class_index["cat"] = 1; | class_index["cat"] = 1; | ||||
| class_index["train"] = 9; | class_index["train"] = 9; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); | std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing(); | ||||
| @@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetters) { | |||||
| class_index["cat"] = 1; | class_index["cat"] = 1; | ||||
| class_index["train"] = 9; | class_index["train"] = 9; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| ds = ds->Batch(2); | ds = ds->Batch(2); | ||||
| @@ -120,7 +120,7 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) { | |||||
| // Create a VOC Dataset | // Create a VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) { | |||||
| // Create a VOC Dataset | // Create a VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds1 = VOC(folder_path, "Classification", "train", {}, false, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds1, nullptr); | EXPECT_NE(ds1, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) { | |||||
| // Expect failure: invalid Manifest input, invalid task | // Expect failure: invalid Manifest input, invalid task | ||||
| EXPECT_EQ(iter1, nullptr); | EXPECT_EQ(iter1, nullptr); | ||||
| std::shared_ptr<Dataset> ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4)); | |||||
| std::shared_ptr<Dataset> ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, std::make_shared<RandomSampler>(false, 4)); | |||||
| EXPECT_NE(ds2, nullptr); | EXPECT_NE(ds2, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -188,7 +188,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) { | |||||
| // Create a VOC Dataset | // Create a VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -240,7 +240,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentationError2) { | |||||
| std::map<std::string, int32_t> class_index; | std::map<std::string, int32_t> class_index; | ||||
| class_index["car"] = 0; | class_index["car"] = 0; | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, std::make_shared<RandomSampler>(false, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestCelebADataset) { | |||||
| // Create a CelebA Dataset | // Create a CelebA Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCelebAData/"; | std::string folder_path = datasets_root_path_ + "/testCelebAData/"; | ||||
| std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", SequentialSampler(0, 2), false, {}); | |||||
| std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", std::make_shared<SequentialSampler>(0, 2), false, {}); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -208,7 +208,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongExtensionFail) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"}); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2), {".JGP"}); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -260,7 +260,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongSamplerFail) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(-2, 5)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(-2, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -274,7 +274,7 @@ TEST_F(MindDataTestPipeline, TestMnistGetDatasetSize) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| EXPECT_EQ(ds->GetDatasetSize(), 20); | EXPECT_EQ(ds->GetDatasetSize(), 20); | ||||
| } | } | ||||
| @@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMnistFailWithWrongDatasetDirFail) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFailWithWrongDatasetDirFail."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFailWithWrongDatasetDirFail."; | ||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::shared_ptr<Dataset> ds = Mnist("", "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Mnist("", "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSize) { | |||||
| std::map<std::string, int32_t> class_index; | std::map<std::string, int32_t> class_index; | ||||
| class_index["class1"] = 111; | class_index["class1"] = 111; | ||||
| class_index["class2"] = 333; | class_index["class2"] = 333; | ||||
| auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index); | |||||
| auto ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, class_index); | |||||
| EXPECT_EQ(ds->GetNumClasses(), 2); | EXPECT_EQ(ds->GetNumClasses(), 2); | ||||
| } | } | ||||
| @@ -320,6 +320,6 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSizeFail) { | |||||
| std::map<std::string, int32_t> class_index; | std::map<std::string, int32_t> class_index; | ||||
| class_index["class1"] = 111; | class_index["class1"] = 111; | ||||
| class_index["wrong class"] = 333; | class_index["wrong class"] = 333; | ||||
| auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index); | |||||
| auto ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, class_index); | |||||
| EXPECT_EQ(ds->GetNumClasses(), -1); | EXPECT_EQ(ds->GetNumClasses(), -1); | ||||
| } | } | ||||
| @@ -34,7 +34,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, sampler_size)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, sampler_size)); | |||||
| ds = ds->SetNumWorkers(2); | ds = ds->SetNumWorkers(2); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -79,7 +79,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size)); | |||||
| ds = ds->SetNumWorkers(3); | ds = ds->SetNumWorkers(3); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -125,7 +125,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size)); | |||||
| ds = ds->SetNumWorkers(3); | ds = ds->SetNumWorkers(3); | ||||
| ds = ds->Repeat(num_repeats); | ds = ds->Repeat(num_repeats); | ||||
| @@ -172,7 +172,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(5, sampler_size)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(5, sampler_size)); | |||||
| ds = ds->Repeat(num_repeats[0]); | ds = ds->Repeat(num_repeats[0]); | ||||
| ds = ds->Repeat(num_repeats[1]); | ds = ds->Repeat(num_repeats[1]); | ||||
| @@ -16,6 +16,7 @@ | |||||
| #include "common/common.h" | #include "common/common.h" | ||||
| #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" | #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" | ||||
| #include "minddata/dataset/include/datasets.h" | #include "minddata/dataset/include/datasets.h" | ||||
| #include <functional> | |||||
| using namespace mindspore::dataset; | using namespace mindspore::dataset; | ||||
| using mindspore::dataset::Tensor; | using mindspore::dataset::Tensor; | ||||
| @@ -25,27 +26,27 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { | |||||
| }; | }; | ||||
| TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { | TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { | ||||
| std::shared_ptr<SamplerObj> sampl = DistributedSampler(2, 1); | |||||
| std::shared_ptr<Sampler> sampl = std::make_shared<DistributedSampler>(2, 1); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl = PKSampler(3); | |||||
| sampl = std::make_shared<PKSampler>(3); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl = RandomSampler(false, 12); | |||||
| sampl = std::make_shared<RandomSampler>(false, 12); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl = SequentialSampler(0, 12); | |||||
| sampl = std::make_shared<SequentialSampler>(0, 12); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; | std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; | ||||
| sampl = WeightedRandomSampler(weights, 12); | |||||
| sampl = std::make_shared<WeightedRandomSampler>(weights, 12); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; | std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; | ||||
| sampl = SubsetSampler(indices); | |||||
| sampl = std::make_shared<SubsetSampler>(indices); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl = SubsetRandomSampler(indices); | |||||
| sampl = std::make_shared<SubsetRandomSampler>(indices); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| @@ -88,47 +89,47 @@ TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { | |||||
| TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { | TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { | ||||
| int64_t num_rows = 30; // dummy variable for number of rows in the dataset | int64_t num_rows = 30; // dummy variable for number of rows in the dataset | ||||
| std::shared_ptr<SamplerObj> sampl = DistributedSampler(2, 1, false, 6); | |||||
| std::shared_ptr<SamplerObj> sampl = std::make_shared<DistributedSamplerObj>(2, 1, false, 6, 1, -1, true); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt; | std::shared_ptr<SamplerRT> sampler_rt; | ||||
| sampl->SamplerBuild(&sampler_rt); | sampl->SamplerBuild(&sampler_rt); | ||||
| EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 6); | EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 6); | ||||
| sampl = PKSampler(3, false); | |||||
| sampl = std::make_shared<PKSamplerObj>(3, false, 0); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl->SamplerBuild(&sampler_rt); | sampl->SamplerBuild(&sampler_rt); | ||||
| EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 30); | EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 30); | ||||
| sampl = RandomSampler(false, 12); | |||||
| sampl = std::make_shared<RandomSamplerObj>(false, 12); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl->SamplerBuild(&sampler_rt); | sampl->SamplerBuild(&sampler_rt); | ||||
| EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); | EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); | ||||
| sampl = SequentialSampler(0, 10); | |||||
| sampl = std::make_shared<SequentialSamplerObj>(0, 10); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl->SamplerBuild(&sampler_rt); | sampl->SamplerBuild(&sampler_rt); | ||||
| EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 10); | EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 10); | ||||
| std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; | std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; | ||||
| sampl = WeightedRandomSampler(weights, 12); | |||||
| sampl = std::make_shared<WeightedRandomSamplerObj>(weights, 12); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl->SamplerBuild(&sampler_rt); | sampl->SamplerBuild(&sampler_rt); | ||||
| EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); | EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); | ||||
| std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21}; | std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21}; | ||||
| sampl = SubsetRandomSampler(indices, 11); | |||||
| sampl = std::make_shared<SubsetRandomSamplerObj>(indices, 11); | |||||
| EXPECT_NE(sampl, nullptr); | EXPECT_NE(sampl, nullptr); | ||||
| sampl->SamplerBuild(&sampler_rt); | sampl->SamplerBuild(&sampler_rt); | ||||
| EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 11); | EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 11); | ||||
| // Testing chains | // Testing chains | ||||
| // Parent and child have num_samples | // Parent and child have num_samples | ||||
| std::shared_ptr<SamplerObj> sampl1 = WeightedRandomSampler(weights, 12); | |||||
| std::shared_ptr<SamplerObj> sampl1 = std::make_shared<WeightedRandomSamplerObj>(weights, 12); | |||||
| EXPECT_NE(sampl1, nullptr); | EXPECT_NE(sampl1, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt1; | std::shared_ptr<SamplerRT> sampler_rt1; | ||||
| sampl1->SamplerBuild(&sampler_rt1); | sampl1->SamplerBuild(&sampler_rt1); | ||||
| std::shared_ptr<SamplerObj> sampl2 = SequentialSampler(0, 10); | |||||
| std::shared_ptr<SamplerObj> sampl2 = std::make_shared<SequentialSamplerObj>(0, 10); | |||||
| EXPECT_NE(sampl2, nullptr); | EXPECT_NE(sampl2, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt2; | std::shared_ptr<SamplerRT> sampler_rt2; | ||||
| sampl2->SamplerBuild(&sampler_rt2); | sampl2->SamplerBuild(&sampler_rt2); | ||||
| @@ -136,12 +137,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { | |||||
| EXPECT_EQ(sampler_rt2->CalculateNumSamples(num_rows), 10); | EXPECT_EQ(sampler_rt2->CalculateNumSamples(num_rows), 10); | ||||
| // Parent doesn't have num_samples | // Parent doesn't have num_samples | ||||
| std::shared_ptr<SamplerObj> sampl3 = WeightedRandomSampler(weights, 12); | |||||
| std::shared_ptr<SamplerObj> sampl3 = std::make_shared<WeightedRandomSamplerObj>(weights, 12); | |||||
| EXPECT_NE(sampl3, nullptr); | EXPECT_NE(sampl3, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt3; | std::shared_ptr<SamplerRT> sampler_rt3; | ||||
| sampl3->SamplerBuild(&sampler_rt3); | sampl3->SamplerBuild(&sampler_rt3); | ||||
| std::shared_ptr<SamplerObj> sampl4 = SubsetRandomSampler(indices); | |||||
| std::shared_ptr<SamplerObj> sampl4 = std::make_shared<SubsetRandomSamplerObj>(indices, 0); | |||||
| EXPECT_NE(sampl4, nullptr); | EXPECT_NE(sampl4, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt4; | std::shared_ptr<SamplerRT> sampler_rt4; | ||||
| sampl4->SamplerBuild(&sampler_rt4); | sampl4->SamplerBuild(&sampler_rt4); | ||||
| @@ -149,12 +150,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { | |||||
| EXPECT_EQ(sampler_rt4->CalculateNumSamples(num_rows), 11); | EXPECT_EQ(sampler_rt4->CalculateNumSamples(num_rows), 11); | ||||
| // Child doesn't have num_samples | // Child doesn't have num_samples | ||||
| std::shared_ptr<SamplerObj> sampl5 = RandomSampler(false); | |||||
| std::shared_ptr<SamplerObj> sampl5 = std::make_shared<RandomSamplerObj>(false, 0); | |||||
| EXPECT_NE(sampl5, nullptr); | EXPECT_NE(sampl5, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt5; | std::shared_ptr<SamplerRT> sampler_rt5; | ||||
| sampl5->SamplerBuild(&sampler_rt5); | sampl5->SamplerBuild(&sampler_rt5); | ||||
| std::shared_ptr<SamplerObj> sampl6 = PKSampler(3, false, 7); | |||||
| std::shared_ptr<SamplerObj> sampl6 = std::make_shared<PKSamplerObj>(3, false, 7); | |||||
| EXPECT_NE(sampl6, nullptr); | EXPECT_NE(sampl6, nullptr); | ||||
| std::shared_ptr<SamplerRT> sampler_rt6; | std::shared_ptr<SamplerRT> sampler_rt6; | ||||
| sampl6->SamplerBuild(&sampler_rt6); | sampl6->SamplerBuild(&sampler_rt6); | ||||
| @@ -164,41 +165,80 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { | |||||
| TEST_F(MindDataTestPipeline, TestSamplersMoveParameters) { | TEST_F(MindDataTestPipeline, TestSamplersMoveParameters) { | ||||
| std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; | std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; | ||||
| std::shared_ptr<SamplerObj> sampl1 = SubsetRandomSampler(indices); | |||||
| std::shared_ptr<SamplerObj> sampl1 = std::make_shared<SubsetRandomSamplerObj>(indices, 0); | |||||
| EXPECT_FALSE(indices.empty()); | EXPECT_FALSE(indices.empty()); | ||||
| std::shared_ptr<SamplerRT> sampler_rt = nullptr; | std::shared_ptr<SamplerRT> sampler_rt = nullptr; | ||||
| sampl1->SamplerBuild(&sampler_rt); | sampl1->SamplerBuild(&sampler_rt); | ||||
| EXPECT_NE(sampler_rt, nullptr); | EXPECT_NE(sampler_rt, nullptr); | ||||
| std::shared_ptr<SamplerObj> sampl2 = SubsetRandomSampler(std::move(indices)); | |||||
| std::shared_ptr<SamplerObj> sampl2 = std::make_shared<SubsetRandomSamplerObj>(std::move(indices), 0); | |||||
| EXPECT_TRUE(indices.empty()); | EXPECT_TRUE(indices.empty()); | ||||
| std::shared_ptr<SamplerRT> sampler_rt2 = nullptr; | std::shared_ptr<SamplerRT> sampler_rt2 = nullptr; | ||||
| sampl2->SamplerBuild(&sampler_rt2); | sampl2->SamplerBuild(&sampler_rt2); | ||||
| EXPECT_NE(sampler_rt, nullptr); | EXPECT_NE(sampler_rt, nullptr); | ||||
| } | } | ||||
| TEST_F(MindDataTestPipeline, TestWeightedRandomSamplerFail) { | |||||
| // weights is empty | |||||
| std::vector<double> weights1 = {}; | |||||
| std::shared_ptr<SamplerObj> sampl1 = WeightedRandomSampler(weights1); | |||||
| EXPECT_EQ(sampl1, nullptr); | |||||
| // weights has negative number | |||||
| std::vector<double> weights2 = {0.5, 0.2, -0.4}; | |||||
| std::shared_ptr<SamplerObj> sampl2 = WeightedRandomSampler(weights2); | |||||
| EXPECT_EQ(sampl2, nullptr); | |||||
| // weights elements are all zero | |||||
| std::vector<double> weights3 = {0, 0, 0}; | |||||
| std::shared_ptr<SamplerObj> sampl3 = WeightedRandomSampler(weights3); | |||||
| EXPECT_EQ(sampl3, nullptr); | |||||
| TEST_F(MindDataTestPipeline, TestNoSamplerSuccess1) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNoSamplerSuccess1."; | |||||
| // Test building a dataset with no sampler provided (defaults to random sampler | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate the dataset and get each row | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_NE(iter, nullptr); | |||||
| std::unordered_map<std::string, mindspore::MSTensor> row; | |||||
| iter->GetNextRow(&row); | |||||
| uint64_t i = 0; | |||||
| while (row.size() != 0) { | |||||
| i++; | |||||
| auto label = row["label"]; | |||||
| iter->GetNextRow(&row); | |||||
| } | |||||
| EXPECT_EQ(i, ds->GetDatasetSize()); | |||||
| iter->Stop(); | |||||
| } | |||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess1) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess1."; | |||||
| // Test basic setting of distributed_sampler | |||||
| // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true | |||||
| std::shared_ptr<Sampler> sampler = std::make_shared<DistributedSampler>(4, 0, false, 0, 0, -1, true); | |||||
| EXPECT_NE(sampler, nullptr); | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate the dataset and get each row | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_NE(iter, nullptr); | |||||
| std::unordered_map<std::string, mindspore::MSTensor> row; | |||||
| iter->GetNextRow(&row); | |||||
| uint64_t i = 0; | |||||
| while (row.size() != 0) { | |||||
| i++; | |||||
| auto label = row["label"]; | |||||
| iter->GetNextRow(&row); | |||||
| } | |||||
| EXPECT_EQ(i, 11); | |||||
| iter->Stop(); | |||||
| } | } | ||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess."; | |||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess2) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess2."; | |||||
| // Test basic setting of distributed_sampler | // Test basic setting of distributed_sampler | ||||
| // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true | // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true | ||||
| std::shared_ptr<SamplerObj> sampler = DistributedSampler(4, 0, false, 0, 0, -1, true); | |||||
| Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, -1, true); | |||||
| EXPECT_NE(sampler, nullptr); | EXPECT_NE(sampler, nullptr); | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| @@ -223,16 +263,102 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) { | |||||
| iter->Stop(); | iter->Stop(); | ||||
| } | } | ||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess3."; | |||||
| // Test basic setting of distributed_sampler | |||||
| // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true | |||||
| DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, -1, true); | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate the dataset and get each row | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_NE(iter, nullptr); | |||||
| std::unordered_map<std::string, mindspore::MSTensor> row; | |||||
| iter->GetNextRow(&row); | |||||
| uint64_t i = 0; | |||||
| while (row.size() != 0) { | |||||
| i++; | |||||
| auto label = row["label"]; | |||||
| iter->GetNextRow(&row); | |||||
| } | |||||
| EXPECT_EQ(i, 11); | |||||
| iter->Stop(); | |||||
| } | |||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1."; | |||||
| // Test basic setting of distributed_sampler | |||||
| // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true | |||||
| // offset=5 which is greater than num_shards=4 --> will fail later | |||||
| std::shared_ptr<Sampler> sampler = std::make_shared<DistributedSampler>(4, 0, false, 0, 0, 5, false); | |||||
| EXPECT_NE(sampler, nullptr); | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate will fail because sampler is not initiated successfully. | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_EQ(iter, nullptr); | |||||
| } | |||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerFail2) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail2."; | |||||
| // Test basic setting of distributed_sampler | |||||
| // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true | |||||
| // offset=5 which is greater than num_shards=4 --> will fail later | |||||
| Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, 5, false); | |||||
| EXPECT_NE(sampler, nullptr); | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate will fail because sampler is not initiated successfully. | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_EQ(iter, nullptr); | |||||
| } | |||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerFail3) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail3."; | |||||
| // Test basic setting of distributed_sampler | |||||
| // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true | |||||
| // offset=5 which is greater than num_shards=4 --> will fail later | |||||
| DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, 5, false); | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate will fail because sampler is not initiated successfully. | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_EQ(iter, nullptr); | |||||
| } | |||||
| TEST_F(MindDataTestPipeline, TestSamplerAddChild) { | TEST_F(MindDataTestPipeline, TestSamplerAddChild) { | ||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSamplerAddChild."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSamplerAddChild."; | ||||
| auto sampler = DistributedSampler(1, 0, false, 5, 0, -1, true); | |||||
| auto sampler = std::make_shared<DistributedSampler>(1, 0, false, 5, 0, -1, true); | |||||
| EXPECT_NE(sampler, nullptr); | EXPECT_NE(sampler, nullptr); | ||||
| auto child_sampler = SequentialSampler(); | |||||
| sampler->AddChildSampler(child_sampler); | |||||
| auto child_sampler = std::make_shared<SequentialSampler>(); | |||||
| EXPECT_NE(child_sampler, nullptr); | EXPECT_NE(child_sampler, nullptr); | ||||
| sampler->AddChild(child_sampler); | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler); | ||||
| @@ -253,12 +379,3 @@ TEST_F(MindDataTestPipeline, TestSamplerAddChild) { | |||||
| EXPECT_EQ(ds->GetDatasetSize(), 5); | EXPECT_EQ(ds->GetDatasetSize(), 5); | ||||
| iter->Stop(); | iter->Stop(); | ||||
| } | } | ||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerFail) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail."; | |||||
| // Test invalid offset setting of distributed_sampler | |||||
| // offset=5 which is greater than num_shards=4 | |||||
| std::shared_ptr<SamplerObj> sampler = DistributedSampler(4, 0, false, 0, 0, 5, false); | |||||
| EXPECT_EQ(sampler, nullptr); | |||||
| } | |||||
| @@ -33,7 +33,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 3)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -68,6 +68,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) { | |||||
| EXPECT_EQ(i, 3); | EXPECT_EQ(i, 3); | ||||
| // Manually terminate the pipeline | // Manually terminate the pipeline | ||||
| iter->Stop(); | iter->Stop(); | ||||
| } | } | ||||
| @@ -77,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Resize: Non-positive size value: -1 at element: 0 | // Resize: Non-positive size value: -1 at element: 0 | ||||
| @@ -100,7 +101,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Compose: transform ops must not be null | // Compose: transform ops must not be null | ||||
| @@ -121,7 +122,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail3) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Compose: transform list must not be empty | // Compose: transform list must not be empty | ||||
| @@ -142,7 +143,7 @@ TEST_F(MindDataTestPipeline, TestDuplicateSuccess) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -182,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| int number_of_classes = 10; | int number_of_classes = 10; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -248,7 +249,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) { | |||||
| TEST_F(MindDataTestPipeline, TestOneHotSuccess2) { | TEST_F(MindDataTestPipeline, TestOneHotSuccess2) { | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -299,7 +300,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // incorrect num_class | // incorrect num_class | ||||
| @@ -320,7 +321,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // incorrect num_class | // incorrect num_class | ||||
| @@ -341,7 +342,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -382,7 +383,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Resize: Non-positive size value: -1 at element: 0 | // Resize: Non-positive size value: -1 at element: 0 | ||||
| @@ -405,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // RandomApply: transform ops must not be null | // RandomApply: transform ops must not be null | ||||
| @@ -426,7 +427,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail3) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // RandomApply: Probability has to be between 0 and 1 | // RandomApply: Probability has to be between 0 and 1 | ||||
| @@ -447,7 +448,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail4) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // RandomApply: transform list must not be empty | // RandomApply: transform list must not be empty | ||||
| @@ -468,7 +469,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -510,7 +511,8 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| RandomSampler sampler = RandomSampler(false, 10); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", sampler); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Resize: Non-positive size value: -1 at element: 0 | // Resize: Non-positive size value: -1 at element: 0 | ||||
| @@ -533,7 +535,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // RandomChoice: transform ops must not be null | // RandomChoice: transform ops must not be null | ||||
| @@ -554,7 +556,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // RandomChoice: transform list must not be empty | // RandomChoice: transform list must not be empty | ||||
| @@ -575,7 +577,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastSuccess) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 1)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 1)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -625,7 +627,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastFail) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // incorrect data type | // incorrect data type | ||||
| @@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -142,7 +142,7 @@ TEST_F(MindDataTestPipeline, TestCenterCrop) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -228,7 +228,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| int number_of_classes = 10; | int number_of_classes = 10; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| int number_of_classes = 10; | int number_of_classes = 10; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -358,7 +358,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -393,7 +393,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -428,7 +428,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail3) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -462,7 +462,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail4) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter."; | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -521,7 +521,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestDecode) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -620,7 +620,7 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -672,7 +672,7 @@ TEST_F(MindDataTestPipeline, TestInvert) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 20)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 20)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -710,7 +710,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -744,7 +744,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail2) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -778,7 +778,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -829,7 +829,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) { | |||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Batch operation on ds | // Create a Batch operation on ds | ||||
| @@ -880,7 +880,7 @@ TEST_F(MindDataTestPipeline, TestNormalize) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -958,7 +958,7 @@ TEST_F(MindDataTestPipeline, TestNormalizePad) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1025,7 +1025,7 @@ TEST_F(MindDataTestPipeline, TestPad) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestBoundingBoxAugmentSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBoundingBoxAugmentSuccess."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBoundingBoxAugmentSuccess."; | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| /* FIXME - Resolve BoundingBoxAugment to properly handle TensorTransform input | /* FIXME - Resolve BoundingBoxAugment to properly handle TensorTransform input | ||||
| @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess1) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess1."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess1."; | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, 1)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 1)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create an iterator over the result of the above dataset | // Create an iterator over the result of the above dataset | ||||
| @@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess2) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess2 with different params."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess2 with different params."; | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 1)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -121,7 +121,7 @@ TEST_F(MindDataTestPipeline, TestResize1) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResize1 with single integer input."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResize1 with single integer input."; | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 6)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -183,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestResizeWithBBoxSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResizeWithBBoxSuccess."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResizeWithBBoxSuccess."; | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestRandomSelectSubpolicySuccess) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| /* FIXME - Resolve RandomSelectSubpolicy to properly handle TensorTransform input | /* FIXME - Resolve RandomSelectSubpolicy to properly handle TensorTransform input | ||||
| @@ -46,7 +46,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -96,7 +96,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -145,7 +145,7 @@ TEST_F(MindDataTestPipeline, TestRandomColor) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -209,7 +209,7 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -303,7 +303,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess."; | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 10)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -366,7 +366,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropFail) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -406,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropWithBboxSuccess."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropWithBboxSuccess."; | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -446,7 +446,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxFail) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -493,7 +493,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomHorizontalFlipWithBBoxSuccess."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomHorizontalFlipWithBBoxSuccess."; | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -531,7 +531,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxFail) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -548,7 +548,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalAndVerticalFlip) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -618,7 +618,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -667,7 +667,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -716,7 +716,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -756,7 +756,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -821,7 +821,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess1) { | |||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -861,7 +861,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess2) { | |||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -921,7 +921,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess1) { | |||||
| // Testing RandomResizedCrop with default values | // Testing RandomResizedCrop with default values | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -960,7 +960,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess2) { | |||||
| // Testing RandomResizedCrop with non-default values | // Testing RandomResizedCrop with non-default values | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1000,7 +1000,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail1) { | |||||
| // This should fail because size has negative value | // This should fail because size has negative value | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1020,7 +1020,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail2) { | |||||
| // This should fail because scale isn't in {min, max} format | // This should fail because scale isn't in {min, max} format | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1040,7 +1040,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail3) { | |||||
| // This should fail because ratio isn't in {min, max} format | // This should fail because ratio isn't in {min, max} format | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1060,7 +1060,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail4) { | |||||
| // This should fail because scale has a size of more than 2 | // This should fail because scale has a size of more than 2 | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1080,7 +1080,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess1) { | |||||
| // Testing RandomResizedCropWithBBox with default values | // Testing RandomResizedCropWithBBox with default values | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1119,7 +1119,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess2) { | |||||
| // Testing RandomResizedCropWithBBox with non-default values | // Testing RandomResizedCropWithBBox with non-default values | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1160,7 +1160,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail1) { | |||||
| // This should fail because size has negative value | // This should fail because size has negative value | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1173,7 +1173,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail2) { | |||||
| // This should fail because scale isn't in {min, max} format | // This should fail because scale isn't in {min, max} format | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1186,7 +1186,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail3) { | |||||
| // This should fail because ratio isn't in {min, max} format | // This should fail because ratio isn't in {min, max} format | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1199,7 +1199,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail4) { | |||||
| // This should fail because scale has a size of more than 2 | // This should fail because scale has a size of more than 2 | ||||
| // Create a Cifar10 Dataset | // Create a Cifar10 Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1212,7 +1212,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotation) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1273,7 +1273,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotationFail) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1309,7 +1309,7 @@ TEST_F(MindDataTestPipeline, TestRandomSharpness) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -1374,7 +1374,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1414,7 +1414,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1484,7 +1484,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxSuccess) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomVerticalFlipWithBBoxSuccess."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomVerticalFlipWithBBoxSuccess."; | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -1522,7 +1522,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxFail) { | |||||
| // FIXME: For error tests, need to check for failure from CreateIterator execution | // FIXME: For error tests, need to check for failure from CreateIterator execution | ||||
| // Create an VOC Dataset | // Create an VOC Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | ||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess1) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 4)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -74,7 +74,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess2) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 6)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 6)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -158,7 +158,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess1) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess1 with single integer input."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess1 with single integer input."; | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 4)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -201,7 +201,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess2) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess2 with (height, width) input."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess2 with (height, width) input."; | ||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 2)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 2)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create SoftDvppDecodeResizeJpeg object with single integer input | // Create SoftDvppDecodeResizeJpeg object with single integer input | ||||
| @@ -78,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create a Repeat operation on ds | // Create a Repeat operation on ds | ||||
| @@ -36,7 +36,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeDisabled) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeDisabled"; | MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeDisabled"; | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| std::shared_ptr<TensorTransform> decode(new vision::Decode()); | std::shared_ptr<TensorTransform> decode(new vision::Decode()); | ||||
| @@ -67,7 +67,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeEnabled) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeEnabled"; | MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeEnabled"; | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| std::shared_ptr<TensorTransform> decode(new vision::Decode()); | std::shared_ptr<TensorTransform> decode(new vision::Decode()); | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * | * | ||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -48,7 +48,7 @@ TEST_F(MindDataTestTreeAdapter, TestSimpleTreeAdapter) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", SequentialSampler(0, 4)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 4)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| ds = ds->Batch(2); | ds = ds->Batch(2); | ||||
| @@ -83,7 +83,7 @@ TEST_F(MindDataTestTreeAdapter, TestTreeAdapterWithRepeat) { | |||||
| // Create a Mnist Dataset | // Create a Mnist Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testMnistData/"; | std::string folder_path = datasets_root_path_ + "/testMnistData/"; | ||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", SequentialSampler(0, 3)); | |||||
| std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 3)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| ds = ds->Batch(2, false); | ds = ds->Batch(2, false); | ||||
| @@ -115,7 +115,7 @@ TEST_F(MindDataTestTreeAdapter, TestProjectMapTreeAdapter) { | |||||
| // Create an ImageFolder Dataset | // Create an ImageFolder Dataset | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, 2)); | |||||
| std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 2)); | |||||
| EXPECT_NE(ds, nullptr); | EXPECT_NE(ds, nullptr); | ||||
| // Create objects for the tensor ops | // Create objects for the tensor ops | ||||
| @@ -49,9 +49,9 @@ TEST_F(MindDataTestTreeModifying, AppendChild) { | |||||
| * ds1 | * ds1 | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ds1->Take(10); | std::shared_ptr<Dataset> ds3 = ds1->Take(10); | ||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| Status rc; | Status rc; | ||||
| @@ -105,9 +105,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ds1->Take(10); | std::shared_ptr<Dataset> ds3 = ds1->Take(10); | ||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ds5->Repeat(4); | std::shared_ptr<Dataset> ds2 = ds5->Repeat(4); | ||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| Status rc; | Status rc; | ||||
| @@ -119,7 +119,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { | |||||
| EXPECT_EQ(rc, Status::OK()); | EXPECT_EQ(rc, Status::OK()); | ||||
| // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | ||||
| std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ||||
| std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode(); | std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode(); | ||||
| std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1]; | std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1]; | ||||
| rc = ds4_node->InsertChildAt(1, ds6_to_insert); | rc = ds4_node->InsertChildAt(1, ds6_to_insert); | ||||
| @@ -132,7 +132,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { | |||||
| EXPECT_EQ(rc, Status::OK()); | EXPECT_EQ(rc, Status::OK()); | ||||
| // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | ||||
| ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ||||
| ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| ds6_to_insert = ds6->IRNode(); | ds6_to_insert = ds6->IRNode(); | ||||
| std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0]; | std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0]; | ||||
| rc = ds4_node->InsertChildAt(0, ds6_to_insert); | rc = ds4_node->InsertChildAt(0, ds6_to_insert); | ||||
| @@ -145,7 +145,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { | |||||
| EXPECT_EQ(rc, Status::OK()); | EXPECT_EQ(rc, Status::OK()); | ||||
| // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | ||||
| ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ||||
| ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| ds6_to_insert = ds6->IRNode(); | ds6_to_insert = ds6->IRNode(); | ||||
| rc = ds4_node->InsertChildAt(2, ds6_to_insert); | rc = ds4_node->InsertChildAt(2, ds6_to_insert); | ||||
| EXPECT_EQ(rc, Status::OK()); | EXPECT_EQ(rc, Status::OK()); | ||||
| @@ -165,9 +165,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ds1->Take(10); | std::shared_ptr<Dataset> ds3 = ds1->Take(10); | ||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ds5->Repeat(4); | std::shared_ptr<Dataset> ds2 = ds5->Repeat(4); | ||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| Status rc; | Status rc; | ||||
| @@ -179,7 +179,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) { | |||||
| EXPECT_EQ(rc, Status::OK()); | EXPECT_EQ(rc, Status::OK()); | ||||
| // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | ||||
| std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ||||
| std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode(); | std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode(); | ||||
| std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0]; | std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0]; | ||||
| std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1]; | std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1]; | ||||
| @@ -193,7 +193,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) { | |||||
| EXPECT_EQ(rc, Status::OK()); | EXPECT_EQ(rc, Status::OK()); | ||||
| // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 | ||||
| ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; | ||||
| ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| ds6_to_insert = ds6->IRNode(); | ds6_to_insert = ds6->IRNode(); | ||||
| ds3_node = ds4_node->Children()[0]; | ds3_node = ds4_node->Children()[0]; | ||||
| ds2_node = ds4_node->Children()[1]; | ds2_node = ds4_node->Children()[1]; | ||||
| @@ -246,8 +246,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove01) { | |||||
| */ | */ | ||||
| // Case 1 | // Case 1 | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ds1->Take(10); | std::shared_ptr<Dataset> ds3 = ds1->Take(10); | ||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| Status rc; | Status rc; | ||||
| @@ -271,8 +271,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove02) { | |||||
| // Case 2 | // Case 2 | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ds1->Take(10); | std::shared_ptr<Dataset> ds3 = ds1->Take(10); | ||||
| std::shared_ptr<Dataset> ds4 = ds3 + ds2; | std::shared_ptr<Dataset> ds4 = ds3 + ds2; | ||||
| Status rc; | Status rc; | ||||
| @@ -296,8 +296,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove03) { | |||||
| // Case 3 | // Case 3 | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ds1->Take(10); | std::shared_ptr<Dataset> ds3 = ds1->Take(10); | ||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| Status rc; | Status rc; | ||||
| @@ -357,11 +357,11 @@ TEST_F(MindDataTestTreeModifying, Drop01) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds8 = ds7->Take(20); | std::shared_ptr<Dataset> ds8 = ds7->Take(20); | ||||
| std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | ||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| std::shared_ptr<Dataset> ds6 = ds4->Take(13); | std::shared_ptr<Dataset> ds6 = ds4->Take(13); | ||||
| std::shared_ptr<Dataset> ds10 = ds9 + ds6; | std::shared_ptr<Dataset> ds10 = ds9 + ds6; | ||||
| @@ -419,11 +419,11 @@ TEST_F(MindDataTestTreeModifying, Drop03) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds8 = ds7->Take(20); | std::shared_ptr<Dataset> ds8 = ds7->Take(20); | ||||
| std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | ||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| std::shared_ptr<Dataset> ds6 = ds4->Take(13); | std::shared_ptr<Dataset> ds6 = ds4->Take(13); | ||||
| std::shared_ptr<Dataset> ds10 = ds9 + ds6; | std::shared_ptr<Dataset> ds10 = ds9 + ds6; | ||||
| @@ -469,14 +469,14 @@ TEST_F(MindDataTestTreeModifying, Drop04) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds8 = ds7->Take(20); | std::shared_ptr<Dataset> ds8 = ds7->Take(20); | ||||
| std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | ||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); | std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); | ||||
| std::shared_ptr<Dataset> ds10 = ds9 + ds6; | std::shared_ptr<Dataset> ds10 = ds9 + ds6; | ||||
| Status rc; | Status rc; | ||||
| @@ -526,13 +526,13 @@ TEST_F(MindDataTestTreeModifying, Drop05) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds8 = ds7->Take(20); | std::shared_ptr<Dataset> ds8 = ds7->Take(20); | ||||
| std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | ||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds4 = ds3->Skip(1); | std::shared_ptr<Dataset> ds4 = ds3->Skip(1); | ||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); | std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); | ||||
| std::shared_ptr<Dataset> ds10 = ds9 + ds6; | std::shared_ptr<Dataset> ds10 = ds9 + ds6; | ||||
| Status rc; | Status rc; | ||||
| @@ -581,14 +581,14 @@ TEST_F(MindDataTestTreeModifying, Drop06) { | |||||
| * | * | ||||
| */ | */ | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data/"; | std::string folder_path = datasets_root_path_ + "/testPK/data/"; | ||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds8 = ds7->Take(20); | std::shared_ptr<Dataset> ds8 = ds7->Take(20); | ||||
| std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | std::shared_ptr<Dataset> ds9 = ds8->Skip(1); | ||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2}); | ||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); | |||||
| std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11)); | |||||
| std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); // ds1 is put after (ds5, ds4)!!! | std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); // ds1 is put after (ds5, ds4)!!! | ||||
| std::shared_ptr<Dataset> ds10 = ds9 + ds6; | std::shared_ptr<Dataset> ds10 = ds9 + ds6; | ||||
| Status rc; | Status rc; | ||||