| @@ -800,7 +800,7 @@ AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vect | |||||
| } | } | ||||
| AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | ||||
| const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler, | |||||
| const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<AlbumNode>(CharToString(dataset_dir), CharToString(data_schema), | auto ds = std::make_shared<AlbumNode>(CharToString(dataset_dir), CharToString(data_schema), | ||||
| @@ -826,8 +826,8 @@ CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::ve | |||||
| SetCharToString(extensions), cache); | SetCharToString(extensions), cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| bool decode, const std::set<std::vector<char>> &extensions, | |||||
| CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | |||||
| const Sampler *sampler, bool decode, const std::set<std::vector<char>> &extensions, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<CelebANode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, decode, | auto ds = std::make_shared<CelebANode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, decode, | ||||
| @@ -850,8 +850,8 @@ Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std:: | |||||
| auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | |||||
| const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| @@ -870,8 +870,8 @@ Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std | |||||
| auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | |||||
| Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | |||||
| const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| @@ -901,7 +901,7 @@ CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, | CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, | ||||
| const std::vector<char> &task, const bool &decode, Sampler *sampler, | |||||
| const std::vector<char> &task, const bool &decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<CocoNode>(CharToString(dataset_dir), CharToString(annotation_file), CharToString(task), | auto ds = std::make_shared<CocoNode>(CharToString(dataset_dir), CharToString(annotation_file), CharToString(task), | ||||
| @@ -942,7 +942,7 @@ ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, boo | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, Sampler *sampler, | |||||
| ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler, | |||||
| const std::set<std::vector<char>> &extensions, | const std::set<std::vector<char>> &extensions, | ||||
| const std::map<std::vector<char>, int32_t> &class_indexing, | const std::map<std::vector<char>, int32_t> &class_indexing, | ||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| @@ -981,7 +981,7 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, | ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, | ||||
| Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing, | |||||
| const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing, | |||||
| bool decode, const std::shared_ptr<DatasetCache> &cache) { | bool decode, const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<ManifestNode>(CharToString(dataset_file), CharToString(usage), sampler_obj, | auto ds = std::make_shared<ManifestNode>(CharToString(dataset_file), CharToString(usage), sampler_obj, | ||||
| @@ -1000,7 +1000,7 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st | |||||
| MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | ||||
| const std::vector<std::vector<char>> &columns_list, | const std::vector<std::vector<char>> &columns_list, | ||||
| const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample, | |||||
| const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| nlohmann::json sample = nullptr; | nlohmann::json sample = nullptr; | ||||
| @@ -1012,8 +1012,8 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | ||||
| const std::vector<std::vector<char>> &columns_list, Sampler *sampler, | |||||
| nlohmann::json *padded_sample, int64_t num_padded) { | |||||
| const std::vector<std::vector<char>> &columns_list, const Sampler *sampler, | |||||
| const nlohmann::json *padded_sample, int64_t num_padded) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| nlohmann::json sample = nullptr; | nlohmann::json sample = nullptr; | ||||
| if (padded_sample) { | if (padded_sample) { | ||||
| @@ -1025,7 +1025,7 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | |||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | ||||
| const std::vector<std::vector<char>> &columns_list, | const std::vector<std::vector<char>> &columns_list, | ||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample, | |||||
| const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto sampler_obj = sampler.get().Parse(); | auto sampler_obj = sampler.get().Parse(); | ||||
| nlohmann::json sample = nullptr; | nlohmann::json sample = nullptr; | ||||
| @@ -1039,7 +1039,7 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file, | |||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | ||||
| const std::vector<std::vector<char>> &columns_list, | const std::vector<std::vector<char>> &columns_list, | ||||
| const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample, | |||||
| const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| nlohmann::json sample = nullptr; | nlohmann::json sample = nullptr; | ||||
| @@ -1052,8 +1052,8 @@ MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_f | |||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | ||||
| const std::vector<std::vector<char>> &columns_list, Sampler *sampler, | |||||
| nlohmann::json *padded_sample, int64_t num_padded) { | |||||
| const std::vector<std::vector<char>> &columns_list, const Sampler *sampler, | |||||
| const nlohmann::json *padded_sample, int64_t num_padded) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| nlohmann::json sample = nullptr; | nlohmann::json sample = nullptr; | ||||
| if (padded_sample) { | if (padded_sample) { | ||||
| @@ -1066,7 +1066,7 @@ MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_f | |||||
| } | } | ||||
| MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | ||||
| const std::vector<std::vector<char>> &columns_list, | const std::vector<std::vector<char>> &columns_list, | ||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample, | |||||
| const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded) { | int64_t num_padded) { | ||||
| auto sampler_obj = sampler.get().Parse(); | auto sampler_obj = sampler.get().Parse(); | ||||
| nlohmann::json sample = nullptr; | nlohmann::json sample = nullptr; | ||||
| @@ -1085,7 +1085,7 @@ MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vect | |||||
| auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | ||||
| ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ir_node_ = std::static_pointer_cast<DatasetNode>(ds); | ||||
| } | } | ||||
| MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache) { | const std::shared_ptr<DatasetCache> &cache) { | ||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); | ||||
| @@ -1118,7 +1118,7 @@ VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<c | |||||
| } | } | ||||
| VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, | VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, | ||||
| const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing, | const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing, | ||||
| bool decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) { | |||||
| auto sampler_obj = sampler ? sampler->Parse() : nullptr; | auto sampler_obj = sampler ? sampler->Parse() : nullptr; | ||||
| auto ds = std::make_shared<VOCNode>(CharToString(dataset_dir), CharToString(task), CharToString(usage), | auto ds = std::make_shared<VOCNode>(CharToString(dataset_dir), CharToString(task), CharToString(usage), | ||||
| MapCharToString(class_indexing), decode, sampler_obj, cache); | MapCharToString(class_indexing), decode, sampler_obj, cache); | ||||
| @@ -31,7 +31,7 @@ DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, boo | |||||
| offset_(offset), | offset_(offset), | ||||
| even_dist_(even_dist) {} | even_dist_(even_dist) {} | ||||
| std::shared_ptr<SamplerObj> DistributedSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> DistributedSampler::Parse() const { | |||||
| return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_, | return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_, | ||||
| even_dist_); | even_dist_); | ||||
| } | } | ||||
| @@ -40,7 +40,7 @@ std::shared_ptr<SamplerObj> DistributedSampler::Parse() { | |||||
| PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) | PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) | ||||
| : num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {} | : num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {} | ||||
| std::shared_ptr<SamplerObj> PKSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> PKSampler::Parse() const { | |||||
| return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_); | return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_); | ||||
| } | } | ||||
| @@ -48,7 +48,7 @@ std::shared_ptr<SamplerObj> PKSampler::Parse() { | |||||
| RandomSampler::RandomSampler(bool replacement, int64_t num_samples) | RandomSampler::RandomSampler(bool replacement, int64_t num_samples) | ||||
| : replacement_(replacement), num_samples_(num_samples) {} | : replacement_(replacement), num_samples_(num_samples) {} | ||||
| std::shared_ptr<SamplerObj> RandomSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> RandomSampler::Parse() const { | |||||
| return std::make_shared<RandomSamplerObj>(replacement_, num_samples_); | return std::make_shared<RandomSamplerObj>(replacement_, num_samples_); | ||||
| } | } | ||||
| @@ -56,7 +56,7 @@ std::shared_ptr<SamplerObj> RandomSampler::Parse() { | |||||
| SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples) | SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples) | ||||
| : start_index_(start_index), num_samples_(num_samples) {} | : start_index_(start_index), num_samples_(num_samples) {} | ||||
| std::shared_ptr<SamplerObj> SequentialSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> SequentialSampler::Parse() const { | |||||
| return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_); | return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_); | ||||
| } | } | ||||
| @@ -64,7 +64,7 @@ std::shared_ptr<SamplerObj> SequentialSampler::Parse() { | |||||
| SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples) | SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples) | ||||
| : indices_(indices), num_samples_(num_samples) {} | : indices_(indices), num_samples_(num_samples) {} | ||||
| std::shared_ptr<SamplerObj> SubsetSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> SubsetSampler::Parse() const { | |||||
| return std::make_shared<SubsetSamplerObj>(indices_, num_samples_); | return std::make_shared<SubsetSamplerObj>(indices_, num_samples_); | ||||
| } | } | ||||
| @@ -72,7 +72,7 @@ std::shared_ptr<SamplerObj> SubsetSampler::Parse() { | |||||
| SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples) | SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples) | ||||
| : SubsetSampler(indices, num_samples) {} | : SubsetSampler(indices, num_samples) {} | ||||
| std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() const { | |||||
| return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_); | return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_); | ||||
| } | } | ||||
| @@ -80,7 +80,7 @@ std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() { | |||||
| WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement) | WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement) | ||||
| : weights_(weights), num_samples_(num_samples), replacement_(replacement) {} | : weights_(weights), num_samples_(num_samples), replacement_(replacement) {} | ||||
| std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() { | |||||
| std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() const { | |||||
| return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_); | return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_); | ||||
| } | } | ||||
| @@ -244,7 +244,6 @@ struct SentencePieceTokenizer::Data { | |||||
| : vocab_path_(CharToString(vocab_path)), out_type_(out_type) {} | : vocab_path_(CharToString(vocab_path)), out_type_(out_type) {} | ||||
| std::shared_ptr<SentencePieceVocab> vocab_; | std::shared_ptr<SentencePieceVocab> vocab_; | ||||
| std::string vocab_path_; | std::string vocab_path_; | ||||
| SPieceTokenizerLoadType load_type_; | |||||
| SPieceTokenizerOutType out_type_; | SPieceTokenizerOutType out_type_; | ||||
| }; | }; | ||||
| @@ -594,7 +594,8 @@ struct RandomSelectSubpolicy::Data { | |||||
| std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_; | std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_; | ||||
| }; | }; | ||||
| RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy) | |||||
| RandomSelectSubpolicy::RandomSelectSubpolicy( | |||||
| const std::vector<std::vector<std::pair<TensorTransform *, double>>> &policy) | |||||
| : data_(std::make_shared<Data>()) { | : data_(std::make_shared<Data>()) { | ||||
| for (int32_t i = 0; i < policy.size(); i++) { | for (int32_t i = 0; i < policy.size(); i++) { | ||||
| std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy; | std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy; | ||||
| @@ -610,7 +611,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<T | |||||
| } | } | ||||
| RandomSelectSubpolicy::RandomSelectSubpolicy( | RandomSelectSubpolicy::RandomSelectSubpolicy( | ||||
| std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy) | |||||
| const std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> &policy) | |||||
| : data_(std::make_shared<Data>()) { | : data_(std::make_shared<Data>()) { | ||||
| for (int32_t i = 0; i < policy.size(); i++) { | for (int32_t i = 0; i < policy.size(); i++) { | ||||
| std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy; | std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy; | ||||
| @@ -626,7 +627,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy( | |||||
| } | } | ||||
| RandomSelectSubpolicy::RandomSelectSubpolicy( | RandomSelectSubpolicy::RandomSelectSubpolicy( | ||||
| std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy) | |||||
| const std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> &policy) | |||||
| : data_(std::make_shared<Data>()) { | : data_(std::make_shared<Data>()) { | ||||
| for (int32_t i = 0; i < policy.size(); i++) { | for (int32_t i = 0; i < policy.size(); i++) { | ||||
| std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy; | std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy; | ||||
| @@ -690,7 +690,7 @@ class AlbumDataset : public Dataset { | |||||
| const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler, | const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | ||||
| const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler, | |||||
| const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | ||||
| const std::vector<std::vector<char>> &column_names, bool decode, | const std::vector<std::vector<char>> &column_names, bool decode, | ||||
| @@ -727,7 +727,8 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | ||||
| const std::vector<std::string> &column_names, bool decode, Sampler *sampler, | |||||
| const std::vector<std::string> &column_names, bool decode, | |||||
| const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema), | return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema), | ||||
| VectorStringToChar(column_names), decode, sampler, cache); | VectorStringToChar(column_names), decode, sampler, cache); | ||||
| @@ -754,7 +755,7 @@ class CelebADataset : public Dataset { | |||||
| explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::shared_ptr<Sampler> &sampler, bool decode, | const std::shared_ptr<Sampler> &sampler, bool decode, | ||||
| const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache); | const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler, | |||||
| bool decode, const std::set<std::vector<char>> &extensions, | bool decode, const std::set<std::vector<char>> &extensions, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| @@ -793,8 +794,9 @@ inline std::shared_ptr<CelebADataset> CelebA( | |||||
| /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). | /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| bool decode = false, const std::set<std::string> &extensions = {}, | |||||
| inline std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, | |||||
| const Sampler *sampler, bool decode = false, | |||||
| const std::set<std::string> &extensions = {}, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<CelebADataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, decode, | return std::make_shared<CelebADataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, decode, | ||||
| SetStringToChar(extensions), cache); | SetStringToChar(extensions), cache); | ||||
| @@ -822,7 +824,7 @@ class Cifar10Dataset : public Dataset { | |||||
| public: | public: | ||||
| explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| @@ -853,7 +855,8 @@ inline std::shared_ptr<Cifar10Dataset> Cifar10( | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, | inline std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, | ||||
| Sampler *sampler, const std::shared_ptr<DatasetCache> &cache = nullptr) { | |||||
| const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | |||||
| return std::make_shared<Cifar10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | return std::make_shared<Cifar10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | ||||
| } | } | ||||
| @@ -874,7 +877,7 @@ class Cifar100Dataset : public Dataset { | |||||
| public: | public: | ||||
| explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| @@ -905,7 +908,7 @@ inline std::shared_ptr<Cifar100Dataset> Cifar100( | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, | inline std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, | ||||
| Sampler *sampler, | |||||
| const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<Cifar100Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | return std::make_shared<Cifar100Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | ||||
| } | } | ||||
| @@ -964,7 +967,7 @@ class CocoDataset : public Dataset { | |||||
| const std::vector<char> &task, const bool &decode, const std::shared_ptr<Sampler> &sampler, | const std::vector<char> &task, const bool &decode, const std::shared_ptr<Sampler> &sampler, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, | CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, | ||||
| const std::vector<char> &task, const bool &decode, Sampler *sampler, | |||||
| const std::vector<char> &task, const bool &decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, | CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file, | ||||
| const std::vector<char> &task, const bool &decode, const std::reference_wrapper<Sampler> sampler, | const std::vector<char> &task, const bool &decode, const std::reference_wrapper<Sampler> sampler, | ||||
| @@ -1015,7 +1018,7 @@ inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const s | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file, | ||||
| const std::string &task, const bool &decode, Sampler *sampler, | |||||
| const std::string &task, const bool &decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<CocoDataset>(StringToChar(dataset_dir), StringToChar(annotation_file), StringToChar(task), | return std::make_shared<CocoDataset>(StringToChar(dataset_dir), StringToChar(annotation_file), StringToChar(task), | ||||
| decode, sampler, cache); | decode, sampler, cache); | ||||
| @@ -1091,7 +1094,7 @@ class ImageFolderDataset : public Dataset { | |||||
| const std::shared_ptr<Sampler> &sampler, const std::set<std::vector<char>> &extensions, | const std::shared_ptr<Sampler> &sampler, const std::set<std::vector<char>> &extensions, | ||||
| const std::map<std::vector<char>, int32_t> &class_indexing, | const std::map<std::vector<char>, int32_t> &class_indexing, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, Sampler *sampler, | |||||
| explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler, | |||||
| const std::set<std::vector<char>> &extensions, | const std::set<std::vector<char>> &extensions, | ||||
| const std::map<std::vector<char>, int32_t> &class_indexing, | const std::map<std::vector<char>, int32_t> &class_indexing, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| @@ -1136,7 +1139,8 @@ inline std::shared_ptr<ImageFolderDataset> ImageFolder( | |||||
| /// \param[in] class_indexing a class name to label map | /// \param[in] class_indexing a class name to label map | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current ImageFolderDataset | /// \return Shared pointer to the current ImageFolderDataset | ||||
| inline std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler, | |||||
| inline std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, | |||||
| const Sampler *sampler, | |||||
| const std::set<std::string> &extensions = {}, | const std::set<std::string> &extensions = {}, | ||||
| const std::map<std::string, int32_t> &class_indexing = {}, | const std::map<std::string, int32_t> &class_indexing = {}, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| @@ -1170,9 +1174,9 @@ class ManifestDataset : public Dataset { | |||||
| const std::shared_ptr<Sampler> &sampler, | const std::shared_ptr<Sampler> &sampler, | ||||
| const std::map<std::vector<char>, int32_t> &class_indexing, bool decode, | const std::map<std::vector<char>, int32_t> &class_indexing, bool decode, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, Sampler *sampler, | |||||
| const std::map<std::vector<char>, int32_t> &class_indexing, bool decode, | |||||
| const std::shared_ptr<DatasetCache> &cache); | |||||
| explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, | |||||
| const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing, | |||||
| bool decode, const std::shared_ptr<DatasetCache> &cache); | |||||
| explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, | explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, | ||||
| const std::reference_wrapper<Sampler> sampler, | const std::reference_wrapper<Sampler> sampler, | ||||
| const std::map<std::vector<char>, int32_t> &class_indexing, bool decode, | const std::map<std::vector<char>, int32_t> &class_indexing, bool decode, | ||||
| @@ -1212,7 +1216,7 @@ inline std::shared_ptr<ManifestDataset> Manifest( | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current ManifestDataset | /// \return Shared pointer to the current ManifestDataset | ||||
| inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, | inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, | ||||
| Sampler *sampler, | |||||
| const Sampler *sampler, | |||||
| const std::map<std::string, int32_t> &class_indexing = {}, | const std::map<std::string, int32_t> &class_indexing = {}, | ||||
| bool decode = false, | bool decode = false, | ||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| @@ -1242,21 +1246,22 @@ inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file | |||||
| class MindDataDataset : public Dataset { | class MindDataDataset : public Dataset { | ||||
| public: | public: | ||||
| explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list, | explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list, | ||||
| const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample, int64_t num_padded); | |||||
| const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded); | |||||
| explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list, | explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list, | ||||
| Sampler *sampler, nlohmann::json *padded_sample, int64_t num_padded); | |||||
| const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded); | |||||
| explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list, | explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list, | ||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample, | |||||
| const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded); | int64_t num_padded); | ||||
| explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | ||||
| const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler, | const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler, | ||||
| nlohmann::json *padded_sample, int64_t num_padded); | |||||
| const nlohmann::json *padded_sample, int64_t num_padded); | |||||
| explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | ||||
| const std::vector<std::vector<char>> &columns_list, Sampler *sampler, | |||||
| nlohmann::json *padded_sample, int64_t num_padded); | |||||
| const std::vector<std::vector<char>> &columns_list, const Sampler *sampler, | |||||
| const nlohmann::json *padded_sample, int64_t num_padded); | |||||
| explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files, | ||||
| const std::vector<std::vector<char>> &columns_list, | const std::vector<std::vector<char>> &columns_list, | ||||
| const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample, | |||||
| const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample, | |||||
| int64_t num_padded); | int64_t num_padded); | ||||
| ~MindDataDataset() = default; | ~MindDataDataset() = default; | ||||
| }; | }; | ||||
| @@ -1290,7 +1295,7 @@ inline std::shared_ptr<MindDataDataset> MindData( | |||||
| /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | ||||
| /// \return Shared pointer to the current MindDataDataset | /// \return Shared pointer to the current MindDataDataset | ||||
| inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, | inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, | ||||
| const std::vector<std::string> &columns_list, Sampler *sampler, | |||||
| const std::vector<std::string> &columns_list, const Sampler *sampler, | |||||
| nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) { | nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) { | ||||
| return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler, | return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler, | ||||
| padded_sample, num_padded); | padded_sample, num_padded); | ||||
| @@ -1340,7 +1345,7 @@ inline std::shared_ptr<MindDataDataset> MindData( | |||||
| /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | /// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. | ||||
| /// \return Shared pointer to the current MindDataDataset | /// \return Shared pointer to the current MindDataDataset | ||||
| inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files, | ||||
| const std::vector<std::string> &columns_list, Sampler *sampler, | |||||
| const std::vector<std::string> &columns_list, const Sampler *sampler, | |||||
| nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) { | nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) { | ||||
| return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler, | return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler, | ||||
| padded_sample, num_padded); | padded_sample, num_padded); | ||||
| @@ -1366,7 +1371,7 @@ class MnistDataset : public Dataset { | |||||
| public: | public: | ||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| @@ -1395,7 +1400,8 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current MnistDataset | /// \return Shared pointer to the current MnistDataset | ||||
| inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, | |||||
| const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | ||||
| } | } | ||||
| @@ -1560,7 +1566,7 @@ class VOCDataset : public Dataset { | |||||
| bool decode, const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | bool decode, const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, | explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, | ||||
| const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing, | const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing, | ||||
| bool decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache); | |||||
| bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache); | |||||
| explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, | explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task, | ||||
| const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing, | const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing, | ||||
| bool decode, const std::reference_wrapper<Sampler> sampler, | bool decode, const std::reference_wrapper<Sampler> sampler, | ||||
| @@ -1607,7 +1613,7 @@ inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std | |||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, | inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, | ||||
| const std::string &usage, const std::map<std::string, int32_t> &class_indexing, | const std::string &usage, const std::map<std::string, int32_t> &class_indexing, | ||||
| bool decode, Sampler *sampler, | |||||
| bool decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<VOCDataset>(StringToChar(dataset_dir), StringToChar(task), StringToChar(usage), | return std::make_shared<VOCDataset>(StringToChar(dataset_dir), StringToChar(task), StringToChar(usage), | ||||
| MapStringToChar(class_indexing), decode, sampler, cache); | MapStringToChar(class_indexing), decode, sampler, cache); | ||||
| @@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this<Sampler> { | |||||
| protected: | protected: | ||||
| /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. | /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. | ||||
| /// \return shared pointer to the newly created TensorOperation. | /// \return shared pointer to the newly created TensorOperation. | ||||
| virtual std::shared_ptr<SamplerObj> Parse() = 0; | |||||
| virtual std::shared_ptr<SamplerObj> Parse() const = 0; | |||||
| std::vector<std::shared_ptr<Sampler>> children_; | std::vector<std::shared_ptr<Sampler>> children_; | ||||
| }; | }; | ||||
| @@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| int64_t num_shards_; | int64_t num_shards_; | ||||
| @@ -120,7 +120,7 @@ class PKSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| int64_t num_val_; | int64_t num_val_; | ||||
| @@ -145,7 +145,7 @@ class RandomSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| bool replacement_; | bool replacement_; | ||||
| @@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| int64_t start_index_; | int64_t start_index_; | ||||
| @@ -193,7 +193,7 @@ class SubsetSampler : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| std::vector<int64_t> indices_; | std::vector<int64_t> indices_; | ||||
| int64_t num_samples_; | int64_t num_samples_; | ||||
| @@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| }; | }; | ||||
| /// \brief A class to represent a Weighted Random Sampler in the data pipeline. | /// \brief A class to represent a Weighted Random Sampler in the data pipeline. | ||||
| @@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| std::vector<double> weights_; | std::vector<double> weights_; | ||||
| @@ -648,16 +648,17 @@ class RandomSelectSubpolicy final : public TensorTransform { | |||||
| public: | public: | ||||
| /// \brief Constructor. | /// \brief Constructor. | ||||
| /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are raw pointers | /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are raw pointers | ||||
| explicit RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy); | |||||
| explicit RandomSelectSubpolicy(const std::vector<std::vector<std::pair<TensorTransform *, double>>> &policy); | |||||
| /// \brief Constructor. | /// \brief Constructor. | ||||
| /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are shared pointers | /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are shared pointers | ||||
| explicit RandomSelectSubpolicy(std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy); | |||||
| explicit RandomSelectSubpolicy( | |||||
| const std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> &policy); | |||||
| /// \brief Constructor. | /// \brief Constructor. | ||||
| /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are object pointers | /// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are object pointers | ||||
| explicit RandomSelectSubpolicy( | explicit RandomSelectSubpolicy( | ||||
| std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy); | |||||
| const std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> &policy); | |||||
| /// \brief Destructor. | /// \brief Destructor. | ||||
| ~RandomSelectSubpolicy() = default; | ~RandomSelectSubpolicy() = default; | ||||
| @@ -456,7 +456,7 @@ class AlbumDataset : public Dataset { | |||||
| const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler, | const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler, | ||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | ||||
| const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler, | |||||
| const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema, | ||||
| const std::vector<std::vector<char>> &column_names, bool decode, | const std::vector<std::vector<char>> &column_names, bool decode, | ||||
| @@ -493,7 +493,8 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const | |||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current Dataset | /// \return Shared pointer to the current Dataset | ||||
| inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema, | ||||
| const std::vector<std::string> &column_names, bool decode, Sampler *sampler, | |||||
| const std::vector<std::string> &column_names, bool decode, | |||||
| const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema), | return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema), | ||||
| VectorStringToChar(column_names), decode, sampler, cache); | VectorStringToChar(column_names), decode, sampler, cache); | ||||
| @@ -519,7 +520,7 @@ class MnistDataset : public Dataset { | |||||
| public: | public: | ||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler, | |||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache); | const std::shared_ptr<DatasetCache> &cache); | ||||
| explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, | ||||
| const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache); | ||||
| @@ -548,7 +549,8 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const | |||||
| /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. | ||||
| /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). | ||||
| /// \return Shared pointer to the current MnistDataset | /// \return Shared pointer to the current MnistDataset | ||||
| inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, | |||||
| inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, | |||||
| const Sampler *sampler, | |||||
| const std::shared_ptr<DatasetCache> &cache = nullptr) { | const std::shared_ptr<DatasetCache> &cache = nullptr) { | ||||
| return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); | ||||
| } | } | ||||
| @@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this<Sampler> { | |||||
| protected: | protected: | ||||
| /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. | /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. | ||||
| /// \return shared pointer to the newly created TensorOperation. | /// \return shared pointer to the newly created TensorOperation. | ||||
| virtual std::shared_ptr<SamplerObj> Parse() = 0; | |||||
| virtual std::shared_ptr<SamplerObj> Parse() const = 0; | |||||
| std::vector<std::shared_ptr<Sampler>> children_; | std::vector<std::shared_ptr<Sampler>> children_; | ||||
| }; | }; | ||||
| @@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| int64_t num_shards_; | int64_t num_shards_; | ||||
| @@ -120,7 +120,7 @@ class PKSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| int64_t num_val_; | int64_t num_val_; | ||||
| @@ -145,7 +145,7 @@ class RandomSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| bool replacement_; | bool replacement_; | ||||
| @@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| int64_t start_index_; | int64_t start_index_; | ||||
| @@ -193,7 +193,7 @@ class SubsetSampler : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| std::vector<int64_t> indices_; | std::vector<int64_t> indices_; | ||||
| int64_t num_samples_; | int64_t num_samples_; | ||||
| @@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| }; | }; | ||||
| /// \brief A class to represent a Weighted Random Sampler in the data pipeline. | /// \brief A class to represent a Weighted Random Sampler in the data pipeline. | ||||
| @@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler { | |||||
| protected: | protected: | ||||
| /// \brief Function to convert a Sampler into an IR SamplerObj. | /// \brief Function to convert a Sampler into an IR SamplerObj. | ||||
| /// \return shared pointer to the newly created SamplerObj. | /// \return shared pointer to the newly created SamplerObj. | ||||
| std::shared_ptr<SamplerObj> Parse() override; | |||||
| std::shared_ptr<SamplerObj> Parse() const override; | |||||
| private: | private: | ||||
| std::vector<double> weights_; | std::vector<double> weights_; | ||||
| @@ -207,6 +207,33 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) { | |||||
| iter->Stop(); | iter->Stop(); | ||||
| } | } | ||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess4) { | |||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess4."; | |||||
| // Test pointer of distributed_sampler | |||||
| SequentialSampler sampler = SequentialSampler(0, 4); | |||||
| // Create an ImageFolder Dataset | |||||
| std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; | |||||
| std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, &sampler); | |||||
| EXPECT_NE(ds, nullptr); | |||||
| // Iterate the dataset and get each row | |||||
| std::shared_ptr<Iterator> iter = ds->CreateIterator(); | |||||
| EXPECT_NE(iter, nullptr); | |||||
| std::unordered_map<std::string, mindspore::MSTensor> row; | |||||
| iter->GetNextRow(&row); | |||||
| uint64_t i = 0; | |||||
| while (row.size() != 0) { | |||||
| i++; | |||||
| auto label = row["label"]; | |||||
| iter->GetNextRow(&row); | |||||
| } | |||||
| EXPECT_EQ(i, 4); | |||||
| iter->Stop(); | |||||
| } | |||||
| TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) { | TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) { | ||||
| MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1."; | MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1."; | ||||
| // Test basic setting of distributed_sampler | // Test basic setting of distributed_sampler | ||||