Browse Source

remove make_unique.h

tags/v0.2.0-alpha
Alexey Shevlyakov 5 years ago
parent
commit
84d780c1a4
60 changed files with 241 additions and 291 deletions
  1. +3
    -4
      mindspore/ccsrc/dataset/api/de_pipeline.cc
  2. +3
    -3
      mindspore/ccsrc/dataset/core/global_context.cc
  3. +2
    -3
      mindspore/ccsrc/dataset/core/tensor.cc
  4. +1
    -1
      mindspore/ccsrc/dataset/engine/data_buffer.cc
  5. +4
    -4
      mindspore/ccsrc/dataset/engine/data_schema.cc
  6. +8
    -8
      mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc
  7. +5
    -5
      mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc
  8. +1
    -1
      mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc
  9. +1
    -1
      mindspore/ccsrc/dataset/engine/datasetops/map_op.cc
  10. +1
    -1
      mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc
  11. +1
    -1
      mindspore/ccsrc/dataset/engine/datasetops/project_op.cc
  12. +2
    -2
      mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc
  13. +6
    -6
      mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
  14. +12
    -12
      mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc
  15. +12
    -12
      mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc
  16. +4
    -4
      mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc
  17. +13
    -13
      mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
  18. +11
    -11
      mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc
  19. +22
    -21
      mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
  20. +13
    -13
      mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc
  21. +3
    -3
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc
  22. +3
    -3
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc
  23. +4
    -4
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc
  24. +1
    -1
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc
  25. +0
    -1
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.h
  26. +3
    -3
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc
  27. +3
    -3
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc
  28. +5
    -5
      mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc
  29. +4
    -5
      mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.cc
  30. +4
    -4
      mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc
  31. +3
    -4
      mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.cc
  32. +18
    -19
      mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
  33. +11
    -11
      mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc
  34. +6
    -8
      mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc
  35. +1
    -1
      mindspore/ccsrc/dataset/engine/db_connector.h
  36. +1
    -1
      mindspore/ccsrc/dataset/engine/execution_tree.cc
  37. +1
    -2
      mindspore/ccsrc/dataset/kernels/image/image_utils.cc
  38. +0
    -1
      mindspore/ccsrc/dataset/kernels/py_func_op.cc
  39. +0
    -1
      mindspore/ccsrc/dataset/util/arena.cc
  40. +1
    -3
      mindspore/ccsrc/dataset/util/circular_pool.cc
  41. +7
    -0
      mindspore/ccsrc/dataset/util/de_error.h
  42. +1
    -2
      mindspore/ccsrc/dataset/util/list.h
  43. +1
    -0
      mindspore/ccsrc/dataset/util/lock.cc
  44. +0
    -1
      mindspore/ccsrc/dataset/util/lock.h
  45. +0
    -37
      mindspore/ccsrc/dataset/util/make_unique.h
  46. +1
    -1
      mindspore/ccsrc/dataset/util/queue.h
  47. +0
    -1
      mindspore/ccsrc/dataset/util/task.h
  48. +1
    -2
      mindspore/ccsrc/device/gpu/blocking_queue.cc
  49. +2
    -3
      mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.h
  50. +2
    -3
      mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.h
  51. +2
    -3
      mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.h
  52. +3
    -4
      mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.h
  53. +2
    -3
      mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.h
  54. +1
    -1
      tests/ut/cpp/dataset/celeba_op_test.cc
  55. +1
    -1
      tests/ut/cpp/dataset/cifar_op_test.cc
  56. +7
    -7
      tests/ut/cpp/dataset/image_folder_op_test.cc
  57. +1
    -1
      tests/ut/cpp/dataset/manifest_op_test.cc
  58. +1
    -1
      tests/ut/cpp/dataset/project_op_test.cc
  59. +1
    -1
      tests/ut/cpp/dataset/stand_alone_samplers_test.cc
  60. +10
    -10
      tests/ut/cpp/dataset/tfReader_op_test.cc

+ 3
- 4
mindspore/ccsrc/dataset/api/de_pipeline.cc View File

@@ -23,7 +23,6 @@
#include "dataset/engine/datasetops/source/image_folder_op.h" #include "dataset/engine/datasetops/source/image_folder_op.h"
#include "dataset/engine/datasetops/source/mnist_op.h" #include "dataset/engine/datasetops/source/mnist_op.h"
#include "dataset/engine/datasetops/source/voc_op.h" #include "dataset/engine/datasetops/source/voc_op.h"
#include "dataset/util/make_unique.h"
#include "dataset/core/tensor.h" #include "dataset/core/tensor.h"
#include "dataset/engine/dataset_iterator.h" #include "dataset/engine/dataset_iterator.h"
#include "dataset/engine/datasetops/source/manifest_op.h" #include "dataset/engine/datasetops/source/manifest_op.h"
@@ -123,7 +122,7 @@ Status DEPipeline::AssignRootNode(const DsOpPtr &dataset_op) { return (tree_->As
Status DEPipeline::LaunchTreeExec() { Status DEPipeline::LaunchTreeExec() {
RETURN_IF_NOT_OK(tree_->Prepare()); RETURN_IF_NOT_OK(tree_->Prepare());
RETURN_IF_NOT_OK(tree_->Launch()); RETURN_IF_NOT_OK(tree_->Launch());
iterator_ = make_unique<DatasetIterator>(tree_);
iterator_ = std::make_unique<DatasetIterator>(tree_);
if (iterator_ == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create an Iterator."); if (iterator_ == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create an Iterator.");
return Status::OK(); return Status::OK();
} }
@@ -311,7 +310,7 @@ Status DEPipeline::ParseStorageOp(const py::dict &args, std::shared_ptr<DatasetO
if (!args["schema"].is_none()) { if (!args["schema"].is_none()) {
(void)builder->SetSchemaFile(ToString(args["schema"])); (void)builder->SetSchemaFile(ToString(args["schema"]));
} else if (!args["schema_json_string"].is_none()) { } else if (!args["schema_json_string"].is_none()) {
std::unique_ptr<DataSchema> schema = make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
std::string s = ToString(args["schema_json_string"]); std::string s = ToString(args["schema_json_string"]);
RETURN_IF_NOT_OK(schema->LoadSchemaString(s, std::vector<std::string>())); RETURN_IF_NOT_OK(schema->LoadSchemaString(s, std::vector<std::string>()));
(void)builder->SetNumRows(schema->num_rows()); (void)builder->SetNumRows(schema->num_rows());
@@ -689,7 +688,7 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr<Dataset
} }
} }
if (schema_exists) { if (schema_exists) {
std::unique_ptr<DataSchema> schema = make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
if (args.contains("schema_file_path")) { if (args.contains("schema_file_path")) {
RETURN_IF_NOT_OK(schema->LoadSchemaFile(ToString(args["schema_file_path"]), columns_to_load)); RETURN_IF_NOT_OK(schema->LoadSchemaFile(ToString(args["schema_file_path"]), columns_to_load));
} else { } else {


+ 3
- 3
mindspore/ccsrc/dataset/core/global_context.cc View File

@@ -55,9 +55,9 @@ Status GlobalContext::Init() {
// For testing we can use Dummy pool instead // For testing we can use Dummy pool instead


// Create some tensor allocators for the different types and hook them into the pool. // Create some tensor allocators for the different types and hook them into the pool.
tensor_allocator_ = mindspore::make_unique<Allocator<Tensor>>(mem_pool_);
cv_tensor_allocator_ = mindspore::make_unique<Allocator<CVTensor>>(mem_pool_);
int_allocator_ = mindspore::make_unique<IntAlloc>(mem_pool_);
tensor_allocator_ = std::make_unique<Allocator<Tensor>>(mem_pool_);
cv_tensor_allocator_ = std::make_unique<Allocator<CVTensor>>(mem_pool_);
int_allocator_ = std::make_unique<IntAlloc>(mem_pool_);
return Status::OK(); return Status::OK();
} }




+ 2
- 3
mindspore/ccsrc/dataset/core/tensor.cc View File

@@ -28,7 +28,6 @@
#include "dataset/core/global_context.h" #include "dataset/core/global_context.h"
#include "dataset/core/pybind_support.h" #include "dataset/core/pybind_support.h"
#include "dataset/core/tensor_shape.h" #include "dataset/core/tensor_shape.h"
#include "dataset/util/make_unique.h"


namespace py = pybind11; namespace py = pybind11;
namespace mindspore { namespace mindspore {
@@ -53,7 +52,7 @@ namespace dataset {
Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) { Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) {
// grab the mem pool from global context and create the allocator for char data area // grab the mem pool from global context and create the allocator for char data area
std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool(); std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool();
data_allocator_ = mindspore::make_unique<Allocator<unsigned char>>(global_pool);
data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool);
} }


Tensor::Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data) : Tensor(shape, type) { Tensor::Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data) : Tensor(shape, type) {
@@ -137,7 +136,7 @@ Status Tensor::CreateTensor(std::shared_ptr<Tensor> *ptr, py::array arr) {
if ((*ptr)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type."); if ((*ptr)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type.");


std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool(); std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool();
(*ptr)->data_allocator_ = mindspore::make_unique<Allocator<unsigned char>>(global_pool);
(*ptr)->data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool);
static_cast<void>((*ptr)->StartAddr()); static_cast<void>((*ptr)->StartAddr());
int64_t byte_size = (*ptr)->SizeInBytes(); int64_t byte_size = (*ptr)->SizeInBytes();
unsigned char *data = static_cast<unsigned char *>(arr.request().ptr); unsigned char *data = static_cast<unsigned char *>(arr.request().ptr);


+ 1
- 1
mindspore/ccsrc/dataset/engine/data_buffer.cc View File

@@ -40,7 +40,7 @@ Status DataBuffer::CreateDataBuffer(
case DatasetType::kTf: { case DatasetType::kTf: {
// This type of buffer is for TF record data. // This type of buffer is for TF record data.
// Allocate derived class version for a TF buffers // Allocate derived class version for a TF buffers
new_data_buffer = mindspore::make_unique<TFBuffer>(id, kDeBFlagNone, storage_client);
new_data_buffer = std::make_unique<TFBuffer>(id, kDeBFlagNone, storage_client);
break; break;
} }
default: { default: {


+ 4
- 4
mindspore/ccsrc/dataset/engine/data_schema.cc View File

@@ -26,8 +26,8 @@
#include "common/utils.h" #include "common/utils.h"
#include "dataset/util/status.h" #include "dataset/util/status.h"
#include "dataset/core/tensor_shape.h" #include "dataset/core/tensor_shape.h"
#include "dataset/util/make_unique.h"
#include "utils/log_adapter.h" #include "utils/log_adapter.h"
#include "dataset/util/de_error.h"


namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {
@@ -58,7 +58,7 @@ ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, Ten
// our shape. Otherwise, set our shape to be empty. // our shape. Otherwise, set our shape to be empty.
if (in_shape != nullptr) { if (in_shape != nullptr) {
// Create a shape and copy construct it into our column's shape. // Create a shape and copy construct it into our column's shape.
tensor_shape_ = mindspore::make_unique<TensorShape>(*in_shape);
tensor_shape_ = std::make_unique<TensorShape>(*in_shape);
} else { } else {
tensor_shape_ = nullptr; tensor_shape_ = nullptr;
} }
@@ -75,7 +75,7 @@ ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, Ten
ColDescriptor::ColDescriptor(const ColDescriptor &in_cd) ColDescriptor::ColDescriptor(const ColDescriptor &in_cd)
: type_(in_cd.type_), rank_(in_cd.rank_), tensor_impl_(in_cd.tensor_impl_), col_name_(in_cd.col_name_) { : type_(in_cd.type_), rank_(in_cd.rank_), tensor_impl_(in_cd.tensor_impl_), col_name_(in_cd.col_name_) {
// If it has a tensor shape, make a copy of it with our own unique_ptr. // If it has a tensor shape, make a copy of it with our own unique_ptr.
tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique<TensorShape>(in_cd.shape()) : nullptr;
tensor_shape_ = in_cd.hasShape() ? std::make_unique<TensorShape>(in_cd.shape()) : nullptr;
} }


// Assignment overload // Assignment overload
@@ -86,7 +86,7 @@ ColDescriptor &ColDescriptor::operator=(const ColDescriptor &in_cd) {
tensor_impl_ = in_cd.tensor_impl_; tensor_impl_ = in_cd.tensor_impl_;
col_name_ = in_cd.col_name_; col_name_ = in_cd.col_name_;
// If it has a tensor shape, make a copy of it with our own unique_ptr. // If it has a tensor shape, make a copy of it with our own unique_ptr.
tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique<TensorShape>(in_cd.shape()) : nullptr;
tensor_shape_ = in_cd.hasShape() ? std::make_unique<TensorShape>(in_cd.shape()) : nullptr;
} }
return *this; return *this;
} }


+ 8
- 8
mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc View File

@@ -59,8 +59,8 @@ Status BatchOp::operator()() {
TaskManager::FindMe()->Post(); TaskManager::FindMe()->Post();
int32_t epoch_num = 0, batch_num = 0, cnt = 0; int32_t epoch_num = 0, batch_num = 0, cnt = 0;
TensorRow new_row; TensorRow new_row;
std::unique_ptr<TensorQTable> table = make_unique<TensorQTable>();
child_iterator_ = mindspore::make_unique<ChildIterator>(this, 0, 0);
std::unique_ptr<TensorQTable> table = std::make_unique<TensorQTable>();
child_iterator_ = std::make_unique<ChildIterator>(this, 0, 0);
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
column_name_map_ = child_iterator_->col_name_id_map(); column_name_map_ = child_iterator_->col_name_id_map();
int32_t cur_batch_size = 0; int32_t cur_batch_size = 0;
@@ -72,7 +72,7 @@ Status BatchOp::operator()() {
if (table->size() == static_cast<size_t>(cur_batch_size)) { if (table->size() == static_cast<size_t>(cur_batch_size)) {
RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack(
std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num))));
table = make_unique<TensorQTable>();
table = std::make_unique<TensorQTable>();
RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num))); RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num)));
} }
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
@@ -82,7 +82,7 @@ Status BatchOp::operator()() {
RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack(
std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num))));
} }
table = make_unique<TensorQTable>(); // this drops when drop == true
table = std::make_unique<TensorQTable>(); // this drops when drop == true
// end of the current epoch, batch_num should start from 0 again // end of the current epoch, batch_num should start from 0 again
batch_num = 0; batch_num = 0;
epoch_num++; epoch_num++;
@@ -153,9 +153,9 @@ Status BatchOp::WorkerEntry(int32_t workerId) {
RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair)); RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair));
while (table_pair.second.ctrl_ != batchCtrl::kQuit) { while (table_pair.second.ctrl_ != batchCtrl::kQuit) {
if (table_pair.second.ctrl_ == batchCtrl::kEOE) { if (table_pair.second.ctrl_ == batchCtrl::kEOE) {
RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
} else if (table_pair.second.ctrl_ == batchCtrl::kEOF) { } else if (table_pair.second.ctrl_ == batchCtrl::kEOF) {
RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
} else if (table_pair.second.ctrl_ == batchCtrl::kNoCtrl) { } else if (table_pair.second.ctrl_ == batchCtrl::kNoCtrl) {
std::unique_ptr<DataBuffer> db = nullptr; std::unique_ptr<DataBuffer> db = nullptr;
RETURN_IF_NOT_OK(MakeBatchedBuffer(std::move(table_pair), &db)); RETURN_IF_NOT_OK(MakeBatchedBuffer(std::move(table_pair), &db));
@@ -170,8 +170,8 @@ Status BatchOp::MakeBatchedBuffer(std::pair<std::unique_ptr<TensorQTable>, CBatc
std::unique_ptr<DataBuffer> *db) { std::unique_ptr<DataBuffer> *db) {
RETURN_UNEXPECTED_IF_NULL(table_pair.first); RETURN_UNEXPECTED_IF_NULL(table_pair.first);
if (!input_column_names_.empty()) RETURN_IF_NOT_OK(MapColumns(&table_pair)); // pass it through pyfunc if (!input_column_names_.empty()) RETURN_IF_NOT_OK(MapColumns(&table_pair)); // pass it through pyfunc
(*db) = make_unique<DataBuffer>(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone);
std::unique_ptr<TensorQTable> dest_table = make_unique<TensorQTable>();
(*db) = std::make_unique<DataBuffer>(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone);
std::unique_ptr<TensorQTable> dest_table = std::make_unique<TensorQTable>();
RETURN_IF_NOT_OK(BatchRows(&table_pair.first, &dest_table, table_pair.first->size())); RETURN_IF_NOT_OK(BatchRows(&table_pair.first, &dest_table, table_pair.first->size()));
(*db)->set_tensor_table(std::move(dest_table)); (*db)->set_tensor_table(std::move(dest_table));
(*db)->set_column_name_map(column_name_map_); (*db)->set_column_name_map(column_name_map_);


+ 5
- 5
mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc View File

@@ -80,9 +80,9 @@ void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) {
MS_LOG(INFO) << "Creating connector in tree operator: " << operator_id_ << ". Producer: " << num_producers MS_LOG(INFO) << "Creating connector in tree operator: " << operator_id_ << ". Producer: " << num_producers
<< ". Consumer: " << num_consumers << "."; << ". Consumer: " << num_consumers << ".";
if (oc_queue_size_ > 0) { if (oc_queue_size_ > 0) {
out_connector_ = mindspore::make_unique<DbConnector>(num_producers, // The number of producers
num_consumers, // Only one consumer (the training App)
oc_queue_size_);
out_connector_ = std::make_unique<DbConnector>(num_producers, // The number of producers
num_consumers, // Only one consumer (the training App)
oc_queue_size_);
} else { } else {
// Some op's may choose not to have an output connector // Some op's may choose not to have an output connector
MS_LOG(INFO) << "Bypassed connector creation for tree operator: " << operator_id_ << "."; MS_LOG(INFO) << "Bypassed connector creation for tree operator: " << operator_id_ << ".";
@@ -149,7 +149,7 @@ Status DatasetOp::GetNextInput(std::unique_ptr<DataBuffer> *p_buffer, int32_t wo
// The base class implementation simply flows the eoe message to output. Derived classes // The base class implementation simply flows the eoe message to output. Derived classes
// may override if they need to perform special eoe handling. // may override if they need to perform special eoe handling.
Status DatasetOp::EoeReceived(int32_t worker_id) { Status DatasetOp::EoeReceived(int32_t worker_id) {
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
return (out_connector_->Add(static_cast<int>(worker_id), std::move(eoe_buffer))); return (out_connector_->Add(static_cast<int>(worker_id), std::move(eoe_buffer)));
} }


@@ -157,7 +157,7 @@ Status DatasetOp::EoeReceived(int32_t worker_id) {
// The base class implementation simply flows the eof message to output. Derived classes // The base class implementation simply flows the eof message to output. Derived classes
// may override if they need to perform special eof handling. // may override if they need to perform special eof handling.
Status DatasetOp::EofReceived(int32_t worker_id) { Status DatasetOp::EofReceived(int32_t worker_id) {
std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer))); return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer)));
} }




+ 1
- 1
mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc View File

@@ -225,7 +225,7 @@ Status DeviceQueueOp::SendDataToCPU() {
MS_LOG(INFO) << "Device queue, sending data to CPU."; MS_LOG(INFO) << "Device queue, sending data to CPU.";
int64_t total_batch = 0; int64_t total_batch = 0;


std::unique_ptr<ChildIterator> child_iterator = mindspore::make_unique<ChildIterator>(this, 0, 0);
std::unique_ptr<ChildIterator> child_iterator = std::make_unique<ChildIterator>(this, 0, 0);
while (!(child_iterator->eof_handled())) { while (!(child_iterator->eof_handled())) {
TensorRow curr_row; TensorRow curr_row;
RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&curr_row)); RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&curr_row));


+ 1
- 1
mindspore/ccsrc/dataset/engine/datasetops/map_op.cc View File

@@ -179,7 +179,7 @@ Status MapOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get(), &keep_input_columns, &to_process_indices, &final_col_name_id_map, RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get(), &keep_input_columns, &to_process_indices, &final_col_name_id_map,
&input_columns, &output_columns)); &input_columns, &output_columns));


std::unique_ptr<TensorQTable> new_tensor_table(mindspore::make_unique<TensorQTable>());
std::unique_ptr<TensorQTable> new_tensor_table(std::make_unique<TensorQTable>());
// Perform the compute function of TensorOp(s) and store the result in new_tensor_table. // Perform the compute function of TensorOp(s) and store the result in new_tensor_table.
RETURN_IF_NOT_OK(WorkerCompute(in_buffer.get(), to_process_indices, new_tensor_table.get(), keep_input_columns, RETURN_IF_NOT_OK(WorkerCompute(in_buffer.get(), to_process_indices, new_tensor_table.get(), keep_input_columns,
&input_columns, &output_columns)); &input_columns, &output_columns));


+ 1
- 1
mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc View File

@@ -48,7 +48,7 @@ Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) {
// Instantiate the worker connector. This is the internal connector, not the operators // Instantiate the worker connector. This is the internal connector, not the operators
// output connector. It has single master consuming from it (num producers is 1), and the number // output connector. It has single master consuming from it (num producers is 1), and the number
// of workers is the defined count from the op. // of workers is the defined count from the op.
worker_connector_ = mindspore::make_unique<DbConnector>(num_workers_, num_producers_, worker_connector_size);
worker_connector_ = std::make_unique<DbConnector>(num_workers_, num_producers_, worker_connector_size);


return Status::OK(); return Status::OK();
} }


+ 1
- 1
mindspore/ccsrc/dataset/engine/datasetops/project_op.cc View File

@@ -79,7 +79,7 @@ Status ProjectOp::Project(std::unique_ptr<DataBuffer> *data_buffer) {
new_column_name_mapping[current_column] = i; new_column_name_mapping[current_column] = i;
projected_column_indices.push_back(column_name_mapping[current_column]); projected_column_indices.push_back(column_name_mapping[current_column]);
} }
std::unique_ptr<TensorQTable> new_tensor_table = mindspore::make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> new_tensor_table = std::make_unique<TensorQTable>();
while ((*data_buffer)->NumRows() > 0) { while ((*data_buffer)->NumRows() > 0) {
TensorRow current_row; TensorRow current_row;
RETURN_IF_NOT_OK((*data_buffer)->PopRow(&current_row)); RETURN_IF_NOT_OK((*data_buffer)->PopRow(&current_row));


+ 2
- 2
mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc View File

@@ -84,13 +84,13 @@ Status RenameOp::operator()() {


// we got eoe, now try again until we get eof // we got eoe, now try again until we get eof
MS_LOG(INFO) << "Rename operator EOE Received."; MS_LOG(INFO) << "Rename operator EOE Received.";
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
MS_LOG(DEBUG) << "Rename operator fetching buffer after EOE."; MS_LOG(DEBUG) << "Rename operator fetching buffer after EOE.";
RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); RETURN_IF_NOT_OK(GetNextInput(&curr_buffer));
} // end of while eof loop } // end of while eof loop


MS_LOG(INFO) << "Rename opeerator EOF Received."; MS_LOG(INFO) << "Rename opeerator EOF Received.";
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
return Status::OK(); return Status::OK();
} }




+ 6
- 6
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc View File

@@ -70,7 +70,7 @@ ShuffleOp::ShuffleOp(int32_t shuffle_size, uint32_t shuffle_seed, int32_t op_con
rng_(shuffle_seed), rng_(shuffle_seed),
buffer_counter_(0), buffer_counter_(0),
rows_per_buffer_(rows_per_buffer), rows_per_buffer_(rows_per_buffer),
shuffle_buffer_(mindspore::make_unique<TensorTable>()),
shuffle_buffer_(std::make_unique<TensorTable>()),
shuffle_last_row_idx_(0), shuffle_last_row_idx_(0),
shuffle_buffer_state_(kShuffleStateInit) {} shuffle_buffer_state_(kShuffleStateInit) {}


@@ -90,7 +90,7 @@ Status ShuffleOp::SelfReset() {
shuffle_seed_ = distribution(random_device); shuffle_seed_ = distribution(random_device);
rng_ = std::mt19937_64(shuffle_seed_); rng_ = std::mt19937_64(shuffle_seed_);
} }
shuffle_buffer_ = mindspore::make_unique<TensorTable>();
shuffle_buffer_ = std::make_unique<TensorTable>();
buffer_counter_ = 0; buffer_counter_ = 0;
shuffle_last_row_idx_ = 0; shuffle_last_row_idx_ = 0;
shuffle_buffer_state_ = kShuffleStateInit; shuffle_buffer_state_ = kShuffleStateInit;
@@ -142,7 +142,7 @@ Status ShuffleOp::operator()() {
// Create the child iterator to fetch our data from. // Create the child iterator to fetch our data from.
int32_t worker_id = 0; int32_t worker_id = 0;
int32_t child_idx = 0; int32_t child_idx = 0;
child_iterator_ = mindspore::make_unique<ChildIterator>(this, worker_id, child_idx);
child_iterator_ = std::make_unique<ChildIterator>(this, worker_id, child_idx);


// Main operator loop // Main operator loop
while (true) { while (true) {
@@ -161,7 +161,7 @@ Status ShuffleOp::operator()() {
// Step 1) // Step 1)
// Create an output tensor table if one is not created yet. // Create an output tensor table if one is not created yet.
if (!new_buffer_table) { if (!new_buffer_table) {
new_buffer_table = mindspore::make_unique<TensorQTable>();
new_buffer_table = std::make_unique<TensorQTable>();
} }


// Step 2) // Step 2)
@@ -176,7 +176,7 @@ Status ShuffleOp::operator()() {
// and send this buffer on it's way up the pipeline. Special case is if this is the // and send this buffer on it's way up the pipeline. Special case is if this is the
// last row then we also send it. // last row then we also send it.
if (new_buffer_table->size() == rows_per_buffer_ || shuffle_last_row_idx_ == 0) { if (new_buffer_table->size() == rows_per_buffer_ || shuffle_last_row_idx_ == 0) {
auto new_buffer = mindspore::make_unique<DataBuffer>(buffer_counter_, DataBuffer::kDeBFlagNone);
auto new_buffer = std::make_unique<DataBuffer>(buffer_counter_, DataBuffer::kDeBFlagNone);
new_buffer->set_tensor_table(std::move(new_buffer_table)); new_buffer->set_tensor_table(std::move(new_buffer_table));
new_buffer->set_column_name_map(column_name_map_); new_buffer->set_column_name_map(column_name_map_);
buffer_counter_++; buffer_counter_++;
@@ -218,7 +218,7 @@ Status ShuffleOp::operator()() {
// Since we overloaded eoeReceived function, we are responsible to flow the EOE up the // Since we overloaded eoeReceived function, we are responsible to flow the EOE up the
// pipepline manually now that we are done draining the shuffle buffer // pipepline manually now that we are done draining the shuffle buffer
MS_LOG(INFO) << "Shuffle operator sending EOE."; MS_LOG(INFO) << "Shuffle operator sending EOE.";
auto eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
auto eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));


// Do not wait for any reset to be flown down from operators above us. // Do not wait for any reset to be flown down from operators above us.


+ 12
- 12
mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc View File

@@ -40,7 +40,7 @@ Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) {
builder_sampler_ = std::make_shared<SequentialSampler>(); builder_sampler_ = std::make_shared<SequentialSampler>();
} }


builder_schema_ = make_unique<DataSchema>();
builder_schema_ = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
// label is like this:0 1 0 0 1...... // label is like this:0 1 0 0 1......
@@ -83,7 +83,7 @@ CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::stri
col_name_map_[data_schema_->column(index).name()] = index; col_name_map_[data_schema_->column(index).name()] = index;
} }


attr_info_queue_ = make_unique<Queue<std::vector<std::string>>>(queue_size);
attr_info_queue_ = std::make_unique<Queue<std::vector<std::string>>>(queue_size);
io_block_queues_.Init(num_workers_, queue_size); io_block_queues_.Init(num_workers_, queue_size);
} }


@@ -311,7 +311,7 @@ Status CelebAOp::AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer) {
row_count++; row_count++;
if (row_count % rows_per_buffer_ == 0) { if (row_count % rows_per_buffer_ == 0) {
RETURN_IF_NOT_OK(io_block_queues_[buff_count++ % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[buff_count++ % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
keys.clear(); keys.clear();
} }
} }
@@ -320,21 +320,21 @@ Status CelebAOp::AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer) {


if (!keys.empty()) { if (!keys.empty()) {
RETURN_IF_NOT_OK(io_block_queues_[(buff_count++) % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[(buff_count++) % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
for (int32_t i = 0; i < num_workers_; i++) { for (int32_t i = 0; i < num_workers_; i++) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[i]->Add(std::move(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))));
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
} }
return Status::OK(); return Status::OK();
} else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
wp_.Clear(); wp_.Clear();
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer)); RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer));
@@ -349,17 +349,17 @@ Status CelebAOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
while (io_block != nullptr) { while (io_block != nullptr) {
if (io_block->eoe() == true) { if (io_block->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
buffer_id = worker_id; buffer_id = worker_id;
} else if (io_block->eof() == true) { } else if (io_block->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
} else { } else {
std::vector<int64_t> keys; std::vector<int64_t> keys;
RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
if (keys.empty()) { if (keys.empty()) {
return Status::OK(); // empty key is a quit signal for workers return Status::OK(); // empty key is a quit signal for workers
} }
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
buffer_id += num_workers_; buffer_id += num_workers_;
@@ -370,7 +370,7 @@ Status CelebAOp::WorkerEntry(int32_t worker_id) {
} }


Status CelebAOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { Status CelebAOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
for (const auto &key : keys) { for (const auto &key : keys) {
TensorRow row; TensorRow row;
RETURN_IF_NOT_OK(LoadTensorRow(image_labels_vec_[key], &row)); RETURN_IF_NOT_OK(LoadTensorRow(image_labels_vec_[key], &row));


+ 12
- 12
mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc View File

@@ -47,7 +47,7 @@ Status CifarOp::Builder::Build(std::shared_ptr<CifarOp> *ptr) {
if (sampler_ == nullptr) { if (sampler_ == nullptr) {
sampler_ = std::make_shared<SequentialSampler>(); sampler_ = std::make_shared<SequentialSampler>();
} }
schema_ = make_unique<DataSchema>();
schema_ = std::make_unique<DataSchema>();
TensorShape scalar = TensorShape::CreateScalar(); TensorShape scalar = TensorShape::CreateScalar();
RETURN_IF_NOT_OK(schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); RETURN_IF_NOT_OK(schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
if (cifar_type_ == kCifar10) { if (cifar_type_ == kCifar10) {
@@ -91,7 +91,7 @@ CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const
col_name_map_[data_schema_->column(i).name()] = i; col_name_map_[data_schema_->column(i).name()] = i;
} }
constexpr uint64_t kUtilQueueSize = 512; constexpr uint64_t kUtilQueueSize = 512;
cifar_raw_data_block_ = make_unique<Queue<std::vector<unsigned char>>>(kUtilQueueSize);
cifar_raw_data_block_ = std::make_unique<Queue<std::vector<unsigned char>>>(kUtilQueueSize);
io_block_queues_.Init(num_workers_, queue_size); io_block_queues_.Init(num_workers_, queue_size);
} }


@@ -114,7 +114,7 @@ Status CifarOp::operator()() {
if (row_cnt_ >= num_samples_) break; // enough row read, break for loop if (row_cnt_ >= num_samples_) break; // enough row read, break for loop
if (row_cnt_ % rows_per_buffer_ == 0) { if (row_cnt_ % rows_per_buffer_ == 0) {
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
keys.clear(); keys.clear();
} }
} }
@@ -122,21 +122,21 @@ Status CifarOp::operator()() {
} }
if (keys.empty() == false) { if (keys.empty() == false) {
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
for (int32_t i = 0; i < num_workers_; i++) { for (int32_t i = 0; i < num_workers_; i++) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
} }
return Status::OK(); return Status::OK();
} else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
wp_.Clear(); wp_.Clear();
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
@@ -169,17 +169,17 @@ Status CifarOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
while (io_block != nullptr) { while (io_block != nullptr) {
if (io_block->eoe() == true) { if (io_block->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
buffer_id = worker_id; buffer_id = worker_id;
} else if (io_block->eof() == true) { } else if (io_block->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
} else { } else {
std::vector<int64_t> keys; std::vector<int64_t> keys;
RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
if (keys.empty() == true) { if (keys.empty() == true) {
return Status::OK(); // empty key is a quit signal for workers return Status::OK(); // empty key is a quit signal for workers
} }
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
buffer_id += num_workers_; buffer_id += num_workers_;
@@ -213,7 +213,7 @@ Status CifarOp::LoadTensorRow(uint64_t index, TensorRow *trow) {


// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
Status CifarOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { Status CifarOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
for (const int64_t &key : keys) { for (const int64_t &key : keys) {
TensorRow trow; TensorRow trow;
RETURN_IF_NOT_OK(LoadTensorRow(key, &trow)); RETURN_IF_NOT_OK(LoadTensorRow(key, &trow));


+ 4
- 4
mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc View File

@@ -173,9 +173,9 @@ Status GeneratorOp::operator()() {
bool eof = false; bool eof = false;
while (!eof) { while (!eof) {
// Create new buffer each iteration // Create new buffer each iteration
fetched_buffer = mindspore::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
fetched_buffer = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
fetched_buffer->set_column_name_map(column_names_map_); fetched_buffer->set_column_name_map(column_names_map_);
std::unique_ptr<TensorQTable> fetched_table = mindspore::make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> fetched_table = std::make_unique<TensorQTable>();
bool eoe = false; bool eoe = false;
{ {
py::gil_scoped_acquire gil_acquire; py::gil_scoped_acquire gil_acquire;
@@ -201,12 +201,12 @@ Status GeneratorOp::operator()() {
if (eoe) { if (eoe) {
// Push out EOE upon StopIteration exception from generator // Push out EOE upon StopIteration exception from generator
MS_LOG(INFO) << "Generator operator sends out EOE."; MS_LOG(INFO) << "Generator operator sends out EOE.";
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
// If last repeat or not repeated, push out EOF and exit master loop // If last repeat or not repeated, push out EOF and exit master loop
MS_LOG(INFO) << "Generator operator sends out EOF."; MS_LOG(INFO) << "Generator operator sends out EOF.";
std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer)));
MS_LOG(INFO) << "Generator operator main execution loop complete."; MS_LOG(INFO) << "Generator operator main execution loop complete.";
eof = true; eof = true;


+ 13
- 13
mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc View File

@@ -39,7 +39,7 @@ Status ImageFolderOp::Builder::Build(std::shared_ptr<ImageFolderOp> *ptr) {
if (builder_sampler_ == nullptr) { if (builder_sampler_ == nullptr) {
builder_sampler_ = std::make_shared<SequentialSampler>(); builder_sampler_ = std::make_shared<SequentialSampler>();
} }
builder_schema_ = make_unique<DataSchema>();
builder_schema_ = std::make_unique<DataSchema>();
TensorShape scalar = TensorShape::CreateScalar(); TensorShape scalar = TensorShape::CreateScalar();
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
@@ -82,8 +82,8 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str
for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) {
col_name_map_[data_schema_->column(i).name()] = i; col_name_map_[data_schema_->column(i).name()] = i;
} }
folder_name_queue_ = make_unique<Queue<std::string>>(num_wkrs * queue_size);
image_name_queue_ = make_unique<Queue<FolderImagesPair>>(num_wkrs * queue_size);
folder_name_queue_ = std::make_unique<Queue<std::string>>(num_wkrs * queue_size);
image_name_queue_ = std::make_unique<Queue<FolderImagesPair>>(num_wkrs * queue_size);
io_block_queues_.Init(num_workers_, queue_size); io_block_queues_.Init(num_workers_, queue_size);
} }


@@ -143,7 +143,7 @@ Status ImageFolderOp::operator()() {
row_cnt_++; row_cnt_++;
if (row_cnt_ % rows_per_buffer_ == 0) { if (row_cnt_ % rows_per_buffer_ == 0) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[buf_cnt_++ % num_workers_]->Add(make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
io_block_queues_[buf_cnt_++ % num_workers_]->Add(std::make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
keys.clear(); keys.clear();
} }
} }
@@ -151,21 +151,21 @@ Status ImageFolderOp::operator()() {
} }
if (keys.empty() == false) { if (keys.empty() == false) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
std::unique_ptr<IOBlock> eoe_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
std::unique_ptr<IOBlock> eof_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
std::unique_ptr<IOBlock> eoe_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
std::unique_ptr<IOBlock> eof_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block)));
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block)));
for (int32_t i = 0; i < num_workers_; ++i) { for (int32_t i = 0; i < num_workers_; ++i) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
} }
return Status::OK(); return Status::OK();
} else { // not the last repeat. Sleep master thread, wait for the wake-up from reset } else { // not the last repeat. Sleep master thread, wait for the wake-up from reset
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
wp_.Clear(); wp_.Clear();
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
@@ -182,15 +182,15 @@ Status ImageFolderOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
while (io_block != nullptr) { while (io_block != nullptr) {
if (io_block->eoe() == true) { if (io_block->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
buffer_id = worker_id; buffer_id = worker_id;
} else if (io_block->eof() == true) { } else if (io_block->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
} else { } else {
std::vector<int64_t> keys; std::vector<int64_t> keys;
RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
buffer_id += num_workers_; buffer_id += num_workers_;
@@ -231,7 +231,7 @@ Status ImageFolderOp::LoadTensorRow(ImageLabelPair pairPtr, TensorRow *trow) {


// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
Status ImageFolderOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { Status ImageFolderOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
TensorRow trow; TensorRow trow;
for (const int64_t &key : keys) { for (const int64_t &key : keys) {
RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow));


+ 11
- 11
mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc View File

@@ -40,7 +40,7 @@ Status ManifestOp::Builder::Build(std::shared_ptr<ManifestOp> *ptr) {
if (builder_sampler_ == nullptr) { if (builder_sampler_ == nullptr) {
builder_sampler_ = std::make_shared<SequentialSampler>(); builder_sampler_ = std::make_shared<SequentialSampler>();
} }
builder_schema_ = make_unique<DataSchema>();
builder_schema_ = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
@@ -105,7 +105,7 @@ Status ManifestOp::AddIoBlock(std::unique_ptr<DataBuffer> *sampler_buffer) {
row_cnt_++; row_cnt_++;
if (row_cnt_ % rows_per_buffer_ == 0) { if (row_cnt_ % rows_per_buffer_ == 0) {
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
keys.clear(); keys.clear();
} }
} }
@@ -113,21 +113,21 @@ Status ManifestOp::AddIoBlock(std::unique_ptr<DataBuffer> *sampler_buffer) {
} }
if (keys.empty() == false) { if (keys.empty() == false) {
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
for (int32_t i = 0; i < num_workers_; i++) { for (int32_t i = 0; i < num_workers_; i++) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
} }
return Status::OK(); return Status::OK();
} else { } else {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
wp_.Clear(); wp_.Clear();
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer)); RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer));
@@ -160,17 +160,17 @@ Status ManifestOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
while (io_block != nullptr) { while (io_block != nullptr) {
if (io_block->eoe() == true) { if (io_block->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
buffer_id = worker_id; buffer_id = worker_id;
} else if (io_block->eof() == true) { } else if (io_block->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
} else { } else {
std::vector<int64_t> keys; std::vector<int64_t> keys;
RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
if (keys.empty()) { if (keys.empty()) {
return Status::OK(); // empty key is a quit signal for workers return Status::OK(); // empty key is a quit signal for workers
} }
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
buffer_id += num_workers_; buffer_id += num_workers_;
@@ -227,7 +227,7 @@ Status ManifestOp::LoadTensorRow(const std::pair<std::string, std::vector<std::s


// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
Status ManifestOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { Status ManifestOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
for (const auto &key : keys) { for (const auto &key : keys) {
TensorRow trow; TensorRow trow;
RETURN_IF_NOT_OK(LoadTensorRow(image_labelname_[static_cast<size_t>(key)], &trow)); RETURN_IF_NOT_OK(LoadTensorRow(image_labelname_[static_cast<size_t>(key)], &trow));


+ 22
- 21
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc View File

@@ -30,7 +30,6 @@
#include "dataset/engine/datasetops/dataset_op.h" #include "dataset/engine/datasetops/dataset_op.h"
#include "dataset/engine/db_connector.h" #include "dataset/engine/db_connector.h"
#include "dataset/engine/execution_tree.h" #include "dataset/engine/execution_tree.h"
#include "dataset/util/make_unique.h"
#include "utils/log_adapter.h" #include "utils/log_adapter.h"


namespace mindspore { namespace mindspore {
@@ -96,18 +95,18 @@ MindRecordOp::MindRecordOp(int32_t num_mind_record_workers, int32_t rows_per_buf
io_blk_queues_.Init(num_workers_, op_connector_queue_size); io_blk_queues_.Init(num_workers_, op_connector_queue_size);
if (!block_reader_) return; if (!block_reader_) return;
for (int32_t i = 0; i < num_workers_; ++i) { for (int32_t i = 0; i < num_workers_; ++i) {
block_buffer_.emplace_back(make_unique<std::vector<ShardTuple>>(std::vector<ShardTuple>{}));
block_buffer_.emplace_back(std::make_unique<std::vector<ShardTuple>>(std::vector<ShardTuple>{}));
} }
} }


// Private helper method to encapsulate some common construction/reset tasks // Private helper method to encapsulate some common construction/reset tasks
Status MindRecordOp::Init() { Status MindRecordOp::Init() {
shard_reader_ = mindspore::make_unique<ShardReader>();
shard_reader_ = std::make_unique<ShardReader>();
auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_); auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_);


CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed."); CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed.");


data_schema_ = mindspore::make_unique<DataSchema>();
data_schema_ = std::make_unique<DataSchema>();


std::vector<std::shared_ptr<Schema>> schema_vec = shard_reader_->get_shard_header()->get_schemas(); std::vector<std::shared_ptr<Schema>> schema_vec = shard_reader_->get_shard_header()->get_schemas();
// check whether schema exists, if so use the first one // check whether schema exists, if so use the first one
@@ -144,7 +143,7 @@ Status MindRecordOp::Init() {
} }


if (!load_all_cols) { if (!load_all_cols) {
std::unique_ptr<DataSchema> tmp_schema = make_unique<DataSchema>();
std::unique_ptr<DataSchema> tmp_schema = std::make_unique<DataSchema>();
for (std::string colname : columns_to_load_) { for (std::string colname : columns_to_load_) {
CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), colname + ": doesn't exist"); CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), colname + ": doesn't exist");
RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname]))); RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname])));
@@ -298,7 +297,7 @@ Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr<T[]> *arr
RETURN_IF_NOT_OK(GetFloat(&value, columns_json[column_name], use_double)); RETURN_IF_NOT_OK(GetFloat(&value, columns_json[column_name], use_double));


*new_shape = TensorShape::CreateScalar(); *new_shape = TensorShape::CreateScalar();
*array_data = mindspore::make_unique<T[]>(1);
*array_data = std::make_unique<T[]>(1);
(*array_data)[0] = value; (*array_data)[0] = value;
} else { } else {
if (column.hasShape()) { if (column.hasShape()) {
@@ -309,7 +308,7 @@ Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr<T[]> *arr
} }


int idx = 0; int idx = 0;
*array_data = mindspore::make_unique<T[]>(new_shape->NumOfElements());
*array_data = std::make_unique<T[]>(new_shape->NumOfElements());
for (auto &element : columns_json[column_name]) { for (auto &element : columns_json[column_name]) {
T value = 0; T value = 0;
RETURN_IF_NOT_OK(GetFloat(&value, element, use_double)); RETURN_IF_NOT_OK(GetFloat(&value, element, use_double));
@@ -350,7 +349,7 @@ Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr<T[]> *array
RETURN_IF_NOT_OK(GetInt(&value, columns_json[column_name])); RETURN_IF_NOT_OK(GetInt(&value, columns_json[column_name]));


*new_shape = TensorShape::CreateScalar(); *new_shape = TensorShape::CreateScalar();
*array_data = mindspore::make_unique<T[]>(1);
*array_data = std::make_unique<T[]>(1);
(*array_data)[0] = value; (*array_data)[0] = value;
} else { } else {
if (column.hasShape()) { if (column.hasShape()) {
@@ -361,7 +360,7 @@ Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr<T[]> *array
} }


int idx = 0; int idx = 0;
*array_data = mindspore::make_unique<T[]>(new_shape->NumOfElements());
*array_data = std::make_unique<T[]>(new_shape->NumOfElements());
for (auto &element : columns_json[column_name]) { for (auto &element : columns_json[column_name]) {
T value = 0; T value = 0;
RETURN_IF_NOT_OK(GetInt(&value, element)); RETURN_IF_NOT_OK(GetInt(&value, element));
@@ -431,12 +430,14 @@ Status MindRecordOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block));
while (io_block != nullptr) { while (io_block != nullptr) {
if (io_block->eoe() == true) { if (io_block->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
RETURN_IF_NOT_OK(
out_connector_->Add(worker_id, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block));
continue; continue;
} }
if (io_block->eof() == true) { if (io_block->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
RETURN_IF_NOT_OK(
out_connector_->Add(worker_id, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block));
continue; continue;
} }
@@ -486,9 +487,9 @@ Status MindRecordOp::WorkerEntry(int32_t worker_id) {


Status MindRecordOp::GetBufferFromReader(std::unique_ptr<DataBuffer> *fetched_buffer, int64_t buffer_id, Status MindRecordOp::GetBufferFromReader(std::unique_ptr<DataBuffer> *fetched_buffer, int64_t buffer_id,
int32_t worker_id) { int32_t worker_id) {
*fetched_buffer = mindspore::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
*fetched_buffer = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
(*fetched_buffer)->set_column_name_map(column_name_mapping_); (*fetched_buffer)->set_column_name_map(column_name_mapping_);
std::unique_ptr<TensorQTable> tensor_table = mindspore::make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> tensor_table = std::make_unique<TensorQTable>();
for (int32_t i = 0; i < rows_per_buffer_; ++i) { for (int32_t i = 0; i < rows_per_buffer_; ++i) {
ShardTuple tupled_buffer; ShardTuple tupled_buffer;
if (block_reader_) { if (block_reader_) {
@@ -597,22 +598,22 @@ Status MindRecordOp::operator()() {
for (int32_t i = 0; i < buffers_needed_; ++i) { for (int32_t i = 0; i < buffers_needed_; ++i) {
if (block_reader_) RETURN_IF_NOT_OK(FetchBlockBuffer(i)); if (block_reader_) RETURN_IF_NOT_OK(FetchBlockBuffer(i));
std::vector<int64_t> keys(1, i); std::vector<int64_t> keys(1, i);
RETURN_IF_NOT_OK(
io_blk_queues_[buf_cnt_++ % num_workers_]->Add(make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
RETURN_IF_NOT_OK(io_blk_queues_[buf_cnt_++ % num_workers_]->Add(
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
for (int32_t i = 0; i < num_workers_; i++) { for (int32_t i = 0; i < num_workers_; i++) {
RETURN_IF_NOT_OK(
io_blk_queues_[i]->Add(std::move(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))));
RETURN_IF_NOT_OK(io_blk_queues_[i]->Add(
std::move(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))));
} }
return Status::OK(); return Status::OK();
} else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));


// reset our buffer count and go to loop again. // reset our buffer count and go to loop again.
RETURN_IF_NOT_OK(shard_reader_wait_post_.Wait()); RETURN_IF_NOT_OK(shard_reader_wait_post_.Wait());
@@ -656,7 +657,7 @@ Status MindRecordOp::LaunchThreadAndInitOp() {
} }


Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *count) { Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *count) {
std::unique_ptr<ShardReader> shard_reader = mindspore::make_unique<ShardReader>();
std::unique_ptr<ShardReader> shard_reader = std::make_unique<ShardReader>();
MSRStatus rc = shard_reader->CountTotalRows(dataset_path, count); MSRStatus rc = shard_reader->CountTotalRows(dataset_path, count);
if (rc == MSRStatus::FAILED) { if (rc == MSRStatus::FAILED) {
RETURN_STATUS_UNEXPECTED("MindRecordOp count total rows failed."); RETURN_STATUS_UNEXPECTED("MindRecordOp count total rows failed.");


+ 13
- 13
mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc View File

@@ -43,7 +43,7 @@ Status MnistOp::Builder::Build(std::shared_ptr<MnistOp> *ptr) {
if (builder_sampler_ == nullptr) { if (builder_sampler_ == nullptr) {
builder_sampler_ = std::make_shared<SequentialSampler>(); builder_sampler_ = std::make_shared<SequentialSampler>();
} }
builder_schema_ = make_unique<DataSchema>();
builder_schema_ = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1)));
TensorShape scalar = TensorShape::CreateScalar(); TensorShape scalar = TensorShape::CreateScalar();
@@ -89,7 +89,7 @@ Status MnistOp::TraversalSampleIds(const std::shared_ptr<Tensor> &sample_ids, st
row_cnt_++; row_cnt_++;
if (row_cnt_ % rows_per_buffer_ == 0) { if (row_cnt_ % rows_per_buffer_ == 0) {
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
keys->clear(); keys->clear();
} }
} }
@@ -115,21 +115,21 @@ Status MnistOp::operator()() {
} }
if (keys.empty() == false) { if (keys.empty() == false) {
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
for (int32_t i = 0; i < num_workers_; ++i) { for (int32_t i = 0; i < num_workers_; ++i) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
} }
return Status::OK(); return Status::OK();
} else { } else {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
wp_.Clear(); wp_.Clear();
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
@@ -145,15 +145,15 @@ Status MnistOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock)); RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock));
while (iOBlock != nullptr) { while (iOBlock != nullptr) {
if (iOBlock->eoe() == true) { if (iOBlock->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
buffer_id = worker_id; buffer_id = worker_id;
} else if (iOBlock->eof() == true) { } else if (iOBlock->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
} else { } else {
std::vector<int64_t> keys; std::vector<int64_t> keys;
RETURN_IF_NOT_OK(iOBlock->GetKeys(&keys)); RETURN_IF_NOT_OK(iOBlock->GetKeys(&keys));
if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
buffer_id += num_workers_; buffer_id += num_workers_;
@@ -178,7 +178,7 @@ Status MnistOp::LoadTensorRow(const MnistLabelPair &mnist_pair, TensorRow *trow)


// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
Status MnistOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { Status MnistOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
TensorRow trow; TensorRow trow;
for (const int64_t &key : keys) { for (const int64_t &key : keys) {
RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow));
@@ -309,8 +309,8 @@ Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *la
CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num_images != num_labels"); CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num_images != num_labels");
// The image size of the Mnist dataset is fixed at [28,28] // The image size of the Mnist dataset is fixed at [28,28]
int64_t size = kMnistImageRows * kMnistImageCols; int64_t size = kMnistImageRows * kMnistImageCols;
auto images_buf = mindspore::make_unique<char[]>(size * num_images);
auto labels_buf = mindspore::make_unique<char[]>(num_images);
auto images_buf = std::make_unique<char[]>(size * num_images);
auto labels_buf = std::make_unique<char[]>(num_images);
if (images_buf == nullptr || labels_buf == nullptr) { if (images_buf == nullptr || labels_buf == nullptr) {
std::string err_msg = "Fail to allocate memory for MNIST Buffer."; std::string err_msg = "Fail to allocate memory for MNIST Buffer.";
MS_LOG(ERROR) << err_msg.c_str(); MS_LOG(ERROR) << err_msg.c_str();


+ 3
- 3
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc View File

@@ -52,9 +52,9 @@ Status DistributedSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer
if (cnt_ > samples_per_buffer_) { if (cnt_ > samples_per_buffer_) {
RETURN_STATUS_UNEXPECTED("Distributed Sampler Error"); RETURN_STATUS_UNEXPECTED("Distributed Sampler Error");
} else if (cnt_ == samples_per_buffer_) { } else if (cnt_ == samples_per_buffer_) {
(*out_buffer) = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
} else { } else {
(*out_buffer) = mindspore::make_unique<DataBuffer>(cnt_, DataBuffer::kDeBFlagNone);
(*out_buffer) = std::make_unique<DataBuffer>(cnt_, DataBuffer::kDeBFlagNone);
std::shared_ptr<Tensor> sample_ids; std::shared_ptr<Tensor> sample_ids;
RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_)); RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_));
int64_t *id_ptr = reinterpret_cast<int64_t *>(sample_ids->StartAddr()); int64_t *id_ptr = reinterpret_cast<int64_t *>(sample_ids->StartAddr());
@@ -63,7 +63,7 @@ Status DistributedSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer
*(id_ptr++) = shuffle_ ? shuffle_vec_[static_cast<size_t>(next_id)] : next_id; *(id_ptr++) = shuffle_ ? shuffle_vec_[static_cast<size_t>(next_id)] : next_id;
} }
TensorRow row(1, sample_ids); TensorRow row(1, sample_ids);
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
} }
return Status::OK(); return Status::OK();
} }


+ 3
- 3
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc View File

@@ -53,9 +53,9 @@ Status PKSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
if (next_id_ > num_pk_samples_ || num_pk_samples_ == 0) { if (next_id_ > num_pk_samples_ || num_pk_samples_ == 0) {
RETURN_STATUS_UNEXPECTED("Index out of bound in PKSampler"); RETURN_STATUS_UNEXPECTED("Index out of bound in PKSampler");
} else if (next_id_ == num_pk_samples_) { } else if (next_id_ == num_pk_samples_) {
(*out_buffer) = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
} else { } else {
(*out_buffer) = mindspore::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
(*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
std::shared_ptr<Tensor> sample_ids; std::shared_ptr<Tensor> sample_ids;
int64_t last_id = int64_t last_id =
(samples_per_buffer_ + next_id_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_ + next_id_; (samples_per_buffer_ + next_id_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_ + next_id_;
@@ -68,7 +68,7 @@ Status PKSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
*(id_ptr++) = samples[rnd_ind]; *(id_ptr++) = samples[rnd_ind];
} }
TensorRow row(1, sample_ids); TensorRow row(1, sample_ids);
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
} }
return Status::OK(); return Status::OK();
} }


+ 4
- 4
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc View File

@@ -32,9 +32,9 @@ Status RandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
if (next_id_ > num_samples_) { if (next_id_ > num_samples_) {
RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error"); RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error");
} else if (next_id_ == num_samples_) { } else if (next_id_ == num_samples_) {
(*out_buffer) = make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
} else { } else {
(*out_buffer) = make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
(*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
std::shared_ptr<Tensor> sampleIds; std::shared_ptr<Tensor> sampleIds;
int64_t last_id = samples_per_buffer_ + next_id_ > num_samples_ ? num_samples_ : samples_per_buffer_ + next_id_; int64_t last_id = samples_per_buffer_ + next_id_ > num_samples_ ? num_samples_ : samples_per_buffer_ + next_id_;
RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_)); RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_));
@@ -44,7 +44,7 @@ Status RandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
} }
next_id_ = last_id; next_id_ = last_id;
TensorRow row(1, sampleIds); TensorRow row(1, sampleIds);
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
} }
return Status::OK(); return Status::OK();
} }
@@ -61,7 +61,7 @@ Status RandomSampler::Init(const RandomAccessOp *op) {
} }
std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_); std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_);
} else { } else {
dist = make_unique<std::uniform_int_distribution<int64_t>>(0, num_rows_ - 1);
dist = std::make_unique<std::uniform_int_distribution<int64_t>>(0, num_rows_ - 1);
} }
rnd_.seed(seed_++); rnd_.seed(seed_++);
return Status::OK(); return Status::OK();


+ 1
- 1
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc View File

@@ -35,7 +35,7 @@ Status Sampler::CreateSamplerTensor(std::shared_ptr<Tensor> *sample_ids, int64_t
} }
if (col_desc_ == nullptr) { if (col_desc_ == nullptr) {
// a ColDescriptor for Tensor that holds SampleIds // a ColDescriptor for Tensor that holds SampleIds
col_desc_ = make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1);
col_desc_ = std::make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1);
} }
TensorShape shape(std::vector<dsize_t>(1, num_elements)); TensorShape shape(std::vector<dsize_t>(1, num_elements));
RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type())); RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type()));


+ 0
- 1
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.h View File

@@ -27,7 +27,6 @@
#include "dataset/engine/data_buffer.h" #include "dataset/engine/data_buffer.h"
#include "dataset/engine/data_schema.h" #include "dataset/engine/data_schema.h"
#include "dataset/engine/datasetops/dataset_op.h" #include "dataset/engine/datasetops/dataset_op.h"
#include "dataset/util/make_unique.h"


namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {


+ 3
- 3
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc View File

@@ -25,9 +25,9 @@ Status SequentialSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer)
if (next_id_ > num_samples_) { if (next_id_ > num_samples_) {
RETURN_STATUS_UNEXPECTED("Sequential Sampler Internal Error"); RETURN_STATUS_UNEXPECTED("Sequential Sampler Internal Error");
} else if (next_id_ == num_samples_) { } else if (next_id_ == num_samples_) {
(*out_buffer) = make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
} else { } else {
(*out_buffer) = make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
(*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
std::shared_ptr<Tensor> sampleIds; std::shared_ptr<Tensor> sampleIds;
int64_t lastId = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_; int64_t lastId = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_;
RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, lastId - next_id_)); RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, lastId - next_id_));
@@ -36,7 +36,7 @@ Status SequentialSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer)
*(idPtr++) = next_id_++; *(idPtr++) = next_id_++;
} }
TensorRow row(1, sampleIds); TensorRow row(1, sampleIds);
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
} }
return Status::OK(); return Status::OK();
} }


+ 3
- 3
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc View File

@@ -64,9 +64,9 @@ Status SubsetRandomSampler::Reset() {
Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
// All samples have been drawn // All samples have been drawn
if (sample_id_ == indices_.size()) { if (sample_id_ == indices_.size()) {
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
} else { } else {
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
std::shared_ptr<Tensor> outputIds; std::shared_ptr<Tensor> outputIds;


int64_t last_id = sample_id_ + samples_per_buffer_; int64_t last_id = sample_id_ + samples_per_buffer_;
@@ -92,7 +92,7 @@ Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffe
} }


// Create a TensorTable from that single tensor and push into DataBuffer // Create a TensorTable from that single tensor and push into DataBuffer
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
} }


return Status::OK(); return Status::OK();


+ 5
- 5
mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc View File

@@ -46,10 +46,10 @@ Status WeightedRandomSampler::Init(const RandomAccessOp *op) {
CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init WeightedRandomSampler"); CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init WeightedRandomSampler");


if (!replacement_) { if (!replacement_) {
exp_dist_ = mindspore::make_unique<std::exponential_distribution<>>(1);
exp_dist_ = std::make_unique<std::exponential_distribution<>>(1);
InitOnePassSampling(); InitOnePassSampling();
} else { } else {
discrete_dist_ = mindspore::make_unique<std::discrete_distribution<int64_t>>(weights_.begin(), weights_.end());
discrete_dist_ = std::make_unique<std::discrete_distribution<int64_t>>(weights_.begin(), weights_.end());
} }


return Status::OK(); return Status::OK();
@@ -96,9 +96,9 @@ Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buf
} }


if (sample_id_ == num_samples_) { if (sample_id_ == num_samples_) {
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
} else { } else {
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
std::shared_ptr<Tensor> outputIds; std::shared_ptr<Tensor> outputIds;


int64_t last_id = sample_id_ + samples_per_buffer_; int64_t last_id = sample_id_ + samples_per_buffer_;
@@ -132,7 +132,7 @@ Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buf
} }


// Create a TensorTable from that single tensor and push into DataBuffer // Create a TensorTable from that single tensor and push into DataBuffer
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
} }


return Status::OK(); return Status::OK();


+ 4
- 5
mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.cc View File

@@ -24,7 +24,6 @@
#include "dataset/engine/datasetops/source/storage_client.h" #include "dataset/engine/datasetops/source/storage_client.h"
#include "dataset/engine/datasetops/source/storage_op.h" #include "dataset/engine/datasetops/source/storage_op.h"
#include "dataset/engine/datasetops/source/tf_client.h" #include "dataset/engine/datasetops/source/tf_client.h"
#include "dataset/util/make_unique.h"
#include "dataset/util/status.h" #include "dataset/util/status.h"


namespace mindspore { namespace mindspore {
@@ -57,7 +56,7 @@ static Status CreateStorageClientSwitch(
case DatasetType::kTf: { case DatasetType::kTf: {
// Construct the derived class TFClient, stored as base class StorageClient // Construct the derived class TFClient, stored as base class StorageClient
store_op->set_rows_per_buffer(32); store_op->set_rows_per_buffer(32);
*out_client = mindspore::make_unique<TFClient>(std::move(schema), store_op);
*out_client = std::make_unique<TFClient>(std::move(schema), store_op);
break; break;
} }
case DatasetType::kUnknown: case DatasetType::kUnknown:
@@ -83,7 +82,7 @@ Status StorageClient::CreateStorageClient(
std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client
// Make a new schema first. This only assigns the dataset type. It does not // Make a new schema first. This only assigns the dataset type. It does not
// create the columns yet. // create the columns yet.
auto new_schema = mindspore::make_unique<DataSchema>();
auto new_schema = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK(new_schema->LoadDatasetType(dataset_schema_path)); RETURN_IF_NOT_OK(new_schema->LoadDatasetType(dataset_schema_path));
RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client));
return Status::OK(); return Status::OK();
@@ -99,7 +98,7 @@ Status StorageClient::CreateStorageClient(
std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client
// The dataset type is passed in by the user. Create an empty schema with only // The dataset type is passed in by the user. Create an empty schema with only
// only the dataset type filled in and then create the client with it. // only the dataset type filled in and then create the client with it.
auto new_schema = mindspore::make_unique<DataSchema>();
auto new_schema = std::make_unique<DataSchema>();
new_schema->set_dataset_type(in_type); new_schema->set_dataset_type(in_type);
RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client));
return Status::OK(); return Status::OK();
@@ -147,7 +146,7 @@ Status StorageClient::AssignDatasetLayout(uint32_t num_rows, // In: Th
// The current schema was just an empty one with only the dataset field populated. // The current schema was just an empty one with only the dataset field populated.
// Let's copy construct a new one that will be a copy of the input schema (releasing the old // Let's copy construct a new one that will be a copy of the input schema (releasing the old
// one) and then set the number of rows that the user requested. // one) and then set the number of rows that the user requested.
data_schema_ = mindspore::make_unique<DataSchema>(schema);
data_schema_ = std::make_unique<DataSchema>(schema);
CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647"); CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647");
num_rows_in_dataset_ = num_rows; num_rows_in_dataset_ = num_rows;




+ 4
- 4
mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc View File

@@ -303,7 +303,7 @@ Status StorageOp::init() {
// For simplicity, we'll make both of them 3 so they are the same size. // For simplicity, we'll make both of them 3 so they are the same size.
int32_t action_queue_size = (buffers_needed / num_workers_) + 1; int32_t action_queue_size = (buffers_needed / num_workers_) + 1;
for (int32_t i = 0; i < num_workers_; ++i) { for (int32_t i = 0; i < num_workers_; ++i) {
auto new_queue = mindspore::make_unique<Queue<int32_t>>(action_queue_size);
auto new_queue = std::make_unique<Queue<int32_t>>(action_queue_size);
action_queue_.push_back(std::move(new_queue)); action_queue_.push_back(std::move(new_queue));
} }
} }
@@ -483,10 +483,10 @@ Status StorageOp::operator()() {
// Post the control message to tell the workers to stop waiting on action queue // Post the control message to tell the workers to stop waiting on action queue
// because we are done! // because we are done!
RETURN_IF_NOT_OK(this->PostEndOfData()); RETURN_IF_NOT_OK(this->PostEndOfData());
std::unique_ptr<DataBuffer> eoeBuffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
std::unique_ptr<DataBuffer> eoeBuffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoeBuffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoeBuffer)));
MS_LOG(INFO) << "StorageOp master: Flow end-of-data eof message."; MS_LOG(INFO) << "StorageOp master: Flow end-of-data eof message.";
std::unique_ptr<DataBuffer> eofBuffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
std::unique_ptr<DataBuffer> eofBuffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eofBuffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eofBuffer)));
MS_LOG(INFO) << "StorageOp master: Main execution loop complete."; MS_LOG(INFO) << "StorageOp master: Main execution loop complete.";
done = true; // while loop exit done = true; // while loop exit
@@ -496,7 +496,7 @@ Status StorageOp::operator()() {
// RepeatOp above us somewhere in the tree will re-init us with the data to fetch again // RepeatOp above us somewhere in the tree will re-init us with the data to fetch again
// once it gets the end-of-epoch message. // once it gets the end-of-epoch message.
MS_LOG(INFO) << "StorageOp master: Flow end-of-epoch eoe message."; MS_LOG(INFO) << "StorageOp master: Flow end-of-epoch eoe message.";
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));


// reset our buffer count and go to loop again. // reset our buffer count and go to loop again.


+ 3
- 4
mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.cc View File

@@ -27,7 +27,6 @@
#include "dataset/core/data_type.h" #include "dataset/core/data_type.h"
#include "dataset/engine/datasetops/source/storage_client.h" #include "dataset/engine/datasetops/source/storage_client.h"
#include "dataset/engine/data_schema.h" #include "dataset/engine/data_schema.h"
#include "dataset/util/make_unique.h"


namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {
@@ -72,7 +71,7 @@ Status TFBuffer::Load() {
} }


// Construct the Tensor table for this buffer. // Construct the Tensor table for this buffer.
tensor_table_ = mindspore::make_unique<TensorQTable>();
tensor_table_ = std::make_unique<TensorQTable>();


// At each position in the tensor table, instantiate the shared pointer to it's Tensor. // At each position in the tensor table, instantiate the shared pointer to it's Tensor.
uint32_t row = 0; uint32_t row = 0;
@@ -272,7 +271,7 @@ Status TFBuffer::LoadFloatList(const ColDescriptor &current_col, const dataengin
// Identify how many values we have and then create a local array of these // Identify how many values we have and then create a local array of these
// to deserialize into // to deserialize into
*num_elements = float_list.value_size(); *num_elements = float_list.value_size();
*float_array = mindspore::make_unique<float[]>(*num_elements);
*float_array = std::make_unique<float[]>(*num_elements);
for (int i = 0; i < float_list.value_size(); i++) { for (int i = 0; i < float_list.value_size(); i++) {
(*float_array)[i] = float_list.value(i); (*float_array)[i] = float_list.value(i);
} }
@@ -294,7 +293,7 @@ Status TFBuffer::LoadIntList(const ColDescriptor &current_col, const dataengine:
// Identify how many values we have and then create a local array of these // Identify how many values we have and then create a local array of these
// to deserialize into // to deserialize into
*num_elements = int64_list.value_size(); *num_elements = int64_list.value_size();
*int_array = mindspore::make_unique<int64_t[]>(*num_elements);
*int_array = std::make_unique<int64_t[]>(*num_elements);
for (int i = 0; i < int64_list.value_size(); i++) { for (int i = 0; i < int64_list.value_size(); i++) {
(*int_array)[i] = int64_list.value(i); (*int_array)[i] = int64_list.value(i);
} }


+ 18
- 19
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc View File

@@ -36,7 +36,6 @@
#include "dataset/engine/db_connector.h" #include "dataset/engine/db_connector.h"
#include "dataset/engine/execution_tree.h" #include "dataset/engine/execution_tree.h"
#include "dataset/engine/jagged_connector.h" #include "dataset/engine/jagged_connector.h"
#include "dataset/util/make_unique.h"
#include "dataset/util/path.h" #include "dataset/util/path.h"
#include "dataset/util/queue.h" #include "dataset/util/queue.h"
#include "dataset/util/random.h" #include "dataset/util/random.h"
@@ -54,7 +53,7 @@ TFReaderOp::Builder::Builder()
builder_op_connector_size_ = config_manager->op_connector_size(); builder_op_connector_size_ = config_manager->op_connector_size();
builder_rows_per_buffer_ = config_manager->rows_per_buffer(); builder_rows_per_buffer_ = config_manager->rows_per_buffer();
builder_shuffle_files_ = false; builder_shuffle_files_ = false;
builder_data_schema_ = make_unique<DataSchema>();
builder_data_schema_ = std::make_unique<DataSchema>();
} }


Status TFReaderOp::Builder::ValidateInputs() const { Status TFReaderOp::Builder::ValidateInputs() const {
@@ -103,7 +102,7 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64
finished_reading_dataset_(false), finished_reading_dataset_(false),
shuffle_files_(shuffle_files), shuffle_files_(shuffle_files),
data_schema_(std::move(data_schema)), data_schema_(std::move(data_schema)),
filename_index_(make_unique<StringIndex>()),
filename_index_(std::make_unique<StringIndex>()),
load_io_block_queue_(true), load_io_block_queue_(true),
load_jagged_connector_(true), load_jagged_connector_(true),
num_rows_(0), num_rows_(0),
@@ -129,7 +128,7 @@ Status TFReaderOp::Init() {
// parallel op base. // parallel op base.
RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_));


jagged_buffer_connector_ = mindspore::make_unique<JaggedConnector>(num_workers_, 1, worker_connector_size_);
jagged_buffer_connector_ = std::make_unique<JaggedConnector>(num_workers_, 1, worker_connector_size_);


// temporary: make size large enough to hold all files + EOE to avoid hangs // temporary: make size large enough to hold all files + EOE to avoid hangs
int32_t safe_queue_size = static_cast<int32_t>(std::ceil(dataset_files_list_.size() / num_workers_)) + 1; int32_t safe_queue_size = static_cast<int32_t>(std::ceil(dataset_files_list_.size() / num_workers_)) + 1;
@@ -229,7 +228,7 @@ Status TFReaderOp::operator()() {
} }


// all workers finished reading for this epoch, and we have read all the data from all workers // all workers finished reading for this epoch, and we have read all the data from all workers
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));


if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
@@ -241,7 +240,7 @@ Status TFReaderOp::operator()() {
} }
} }


std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer)));


RETURN_IF_NOT_OK(PostEndOfData()); RETURN_IF_NOT_OK(PostEndOfData());
@@ -274,7 +273,7 @@ Status TFReaderOp::WorkerEntry(int32_t worker_id) {
MS_LOG(INFO) << "TFReader operator worker " << worker_id << " loaded file " << filename << "."; MS_LOG(INFO) << "TFReader operator worker " << worker_id << " loaded file " << filename << ".";
} }
} else { } else {
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(1, DataBuffer::kDeBFlagEOE);
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(1, DataBuffer::kDeBFlagEOE);
RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(eoe_buffer))); RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(eoe_buffer)));
} }


@@ -288,7 +287,7 @@ Status TFReaderOp::WorkerEntry(int32_t worker_id) {
// When the worker pops this control indicator, it will shut itself down gracefully. // When the worker pops this control indicator, it will shut itself down gracefully.
Status TFReaderOp::PostEndOfData() { Status TFReaderOp::PostEndOfData() {
for (int i = 0; i < num_workers_; ++i) { for (int i = 0; i < num_workers_; ++i) {
std::unique_ptr<FilenameBlock> eof = mindspore::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEof);
std::unique_ptr<FilenameBlock> eof = std::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEof);
RETURN_IF_NOT_OK(PushIoBlockQueue(i, std::move(eof))); RETURN_IF_NOT_OK(PushIoBlockQueue(i, std::move(eof)));
} }


@@ -299,7 +298,7 @@ Status TFReaderOp::PostEndOfData() {
// pops this control indicator, it will wait until the next epoch starts and then resume execution. // pops this control indicator, it will wait until the next epoch starts and then resume execution.
Status TFReaderOp::PostEndOfEpoch(int32_t queue_index) { Status TFReaderOp::PostEndOfEpoch(int32_t queue_index) {
for (int i = 0; i < num_workers_; ++i) { for (int i = 0; i < num_workers_; ++i) {
std::unique_ptr<FilenameBlock> eoe = mindspore::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEoe);
std::unique_ptr<FilenameBlock> eoe = std::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEoe);
RETURN_IF_NOT_OK(PushIoBlockQueue((queue_index + i) % num_workers_, std::move(eoe))); RETURN_IF_NOT_OK(PushIoBlockQueue((queue_index + i) % num_workers_, std::move(eoe)));
} }


@@ -358,7 +357,7 @@ Status TFReaderOp::FillIOBlockShuffle(const std::vector<int64_t> &i_keys) {
} }
if (!equal_rows_per_shard_) { if (!equal_rows_per_shard_) {
if (key_index++ % num_devices_ == device_id_) { if (key_index++ % num_devices_ == device_id_) {
auto ioBlock = make_unique<FilenameBlock>(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
auto ioBlock = std::make_unique<FilenameBlock>(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
queue_index = (queue_index + 1) % num_workers_; queue_index = (queue_index + 1) % num_workers_;
} }
@@ -367,7 +366,7 @@ Status TFReaderOp::FillIOBlockShuffle(const std::vector<int64_t> &i_keys) {
auto file_it = filename_index_->Search(*it); auto file_it = filename_index_->Search(*it);
std::string file_name = file_it.value(); std::string file_name = file_it.value();
if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) {
auto ioBlock = make_unique<FilenameBlock>(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone);
auto ioBlock = std::make_unique<FilenameBlock>(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone);
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
MS_LOG(DEBUG) << "File name " << *it << " start offset " << start_offset << " end_offset " << end_offset; MS_LOG(DEBUG) << "File name " << *it << " start offset " << start_offset << " end_offset " << end_offset;
queue_index = (queue_index + 1) % num_workers_; queue_index = (queue_index + 1) % num_workers_;
@@ -404,14 +403,15 @@ Status TFReaderOp::FillIOBlockNoShuffle() {
} }
if (!equal_rows_per_shard_) { if (!equal_rows_per_shard_) {
if (key_index++ % num_devices_ == device_id_) { if (key_index++ % num_devices_ == device_id_) {
auto ioBlock = make_unique<FilenameBlock>(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
auto ioBlock =
std::make_unique<FilenameBlock>(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
queue_index = (queue_index + 1) % num_workers_; queue_index = (queue_index + 1) % num_workers_;
} }
} else { } else {
std::string file_name = it.value(); std::string file_name = it.value();
if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) {
auto ioBlock = make_unique<FilenameBlock>(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone);
auto ioBlock = std::make_unique<FilenameBlock>(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone);
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
queue_index = (queue_index + 1) % num_workers_; queue_index = (queue_index + 1) % num_workers_;
} }
@@ -490,14 +490,13 @@ Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_off


int64_t rows_read = 0; int64_t rows_read = 0;
int64_t rows_total = 0; int64_t rows_total = 0;
std::unique_ptr<DataBuffer> current_buffer =
mindspore::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
std::unique_ptr<DataBuffer> current_buffer = std::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
std::unordered_map<std::string, int32_t> column_name_map; std::unordered_map<std::string, int32_t> column_name_map;
for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) {
column_name_map[data_schema_->column(i).name()] = i; column_name_map[data_schema_->column(i).name()] = i;
} }
current_buffer->set_column_name_map(column_name_map); current_buffer->set_column_name_map(column_name_map);
std::unique_ptr<TensorQTable> new_tensor_table = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> new_tensor_table = std::make_unique<TensorQTable>();


while (reader.peek() != EOF) { while (reader.peek() != EOF) {
if (!load_jagged_connector_) { if (!load_jagged_connector_) {
@@ -532,9 +531,9 @@ Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_off
current_buffer->set_tensor_table(std::move(new_tensor_table)); current_buffer->set_tensor_table(std::move(new_tensor_table));
RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer))); RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer)));


current_buffer = make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
current_buffer = std::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
current_buffer->set_column_name_map(column_name_map); current_buffer->set_column_name_map(column_name_map);
new_tensor_table = make_unique<TensorQTable>();
new_tensor_table = std::make_unique<TensorQTable>();
rows_read = 0; rows_read = 0;
} }
} }
@@ -742,7 +741,7 @@ Status TFReaderOp::LoadFloatList(const ColDescriptor &current_col, const dataeng
// Identify how many values we have and then create a local array of these // Identify how many values we have and then create a local array of these
// to deserialize into // to deserialize into
*num_elements = float_list.value_size(); *num_elements = float_list.value_size();
*float_array = mindspore::make_unique<float[]>(*num_elements);
*float_array = std::make_unique<float[]>(*num_elements);
for (int i = 0; i < float_list.value_size(); ++i) { for (int i = 0; i < float_list.value_size(); ++i) {
(*float_array)[i] = float_list.value(i); (*float_array)[i] = float_list.value(i);
} }


+ 11
- 11
mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc View File

@@ -38,7 +38,7 @@ Status VOCOp::Builder::Build(std::shared_ptr<VOCOp> *ptr) {
if (builder_sampler_ == nullptr) { if (builder_sampler_ == nullptr) {
builder_sampler_ = std::make_shared<SequentialSampler>(); builder_sampler_ = std::make_shared<SequentialSampler>();
} }
builder_schema_ = make_unique<DataSchema>();
builder_schema_ = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
@@ -85,7 +85,7 @@ Status VOCOp::TraverseSampleIds(const std::shared_ptr<Tensor> &sample_ids, std::
row_cnt_++; row_cnt_++;
if (row_cnt_ % rows_per_buffer_ == 0) { if (row_cnt_ % rows_per_buffer_ == 0) {
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
keys->clear(); keys->clear();
} }
} }
@@ -110,21 +110,21 @@ Status VOCOp::operator()() {
} }
if (keys.empty() == false) { if (keys.empty() == false) {
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
} }
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
std::unique_ptr<IOBlock> eoe_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
std::unique_ptr<IOBlock> eof_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
std::unique_ptr<IOBlock> eoe_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
std::unique_ptr<IOBlock> eof_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block)));
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block)));
for (int32_t i = 0; i < num_workers_; i++) { for (int32_t i = 0; i < num_workers_; i++) {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
} }
return Status::OK(); return Status::OK();
} else { } else {
RETURN_IF_NOT_OK( RETURN_IF_NOT_OK(
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
RETURN_IF_NOT_OK(wp_.Wait()); RETURN_IF_NOT_OK(wp_.Wait());
wp_.Clear(); wp_.Clear();
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
@@ -164,7 +164,7 @@ Status VOCOp::LoadTensorRow(const std::string &image_id, TensorRow *trow) {
} }


Status VOCOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { Status VOCOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
TensorRow trow; TensorRow trow;
for (const uint64_t &key : keys) { for (const uint64_t &key : keys) {
RETURN_IF_NOT_OK(this->LoadTensorRow(image_ids_[key], &trow)); RETURN_IF_NOT_OK(this->LoadTensorRow(image_ids_[key], &trow));
@@ -182,15 +182,15 @@ Status VOCOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
while (io_block != nullptr) { while (io_block != nullptr) {
if (io_block->eoe() == true) { if (io_block->eoe() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
buffer_id = worker_id; buffer_id = worker_id;
} else if (io_block->eof() == true) { } else if (io_block->eof() == true) {
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
} else { } else {
std::vector<int64_t> keys; std::vector<int64_t> keys;
RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
if (keys.empty() == true) return Status::OK(); if (keys.empty() == true) return Status::OK();
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
buffer_id += num_workers_; buffer_id += num_workers_;


+ 6
- 8
mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc View File

@@ -65,13 +65,13 @@ Status ZipOp::operator()() {
// initialize the iterators // initialize the iterators
for (int32_t i = 0; i < children_num_; ++i) { for (int32_t i = 0; i < children_num_; ++i) {
// magic number 0 since Zip is not a parallel Op // magic number 0 since Zip is not a parallel Op
child_iterators_.push_back(mindspore::make_unique<ChildIterator>(this, 0, i));
child_iterators_.push_back(std::make_unique<ChildIterator>(this, 0, i));
} }


// Loop until eof is true // Loop until eof is true
while (!eof_) { while (!eof_) {
// Create tensor table and prepare it by fetching and packing the first zipped row into it. // Create tensor table and prepare it by fetching and packing the first zipped row into it.
std::unique_ptr<TensorQTable> curr_table = mindspore::make_unique<TensorQTable>();
std::unique_ptr<TensorQTable> curr_table = std::make_unique<TensorQTable>();
RETURN_IF_NOT_OK(prepare(curr_table.get())); RETURN_IF_NOT_OK(prepare(curr_table.get()));


// If an eof got picked up during the above prepare, then we're done // If an eof got picked up during the above prepare, then we're done
@@ -81,7 +81,7 @@ Status ZipOp::operator()() {
while (!draining_) { while (!draining_) {
// 1. If a previous loop iteration sent the current table out, then create a new one. // 1. If a previous loop iteration sent the current table out, then create a new one.
if (curr_table == nullptr) { if (curr_table == nullptr) {
curr_table = mindspore::make_unique<TensorQTable>();
curr_table = std::make_unique<TensorQTable>();
} }


// 2 fill the table. Note: draining mode might get turned on if any of the child inputs were done // 2 fill the table. Note: draining mode might get turned on if any of the child inputs were done
@@ -89,8 +89,7 @@ Status ZipOp::operator()() {


// 3 create and update buffer and send it to the out connector // 3 create and update buffer and send it to the out connector
if (!curr_table->empty()) { if (!curr_table->empty()) {
std::unique_ptr<DataBuffer> curr_buffer =
mindspore::make_unique<DataBuffer>(buffer_id_, DataBuffer::kDeBFlagNone);
std::unique_ptr<DataBuffer> curr_buffer = std::make_unique<DataBuffer>(buffer_id_, DataBuffer::kDeBFlagNone);
curr_buffer->set_tensor_table(std::move(curr_table)); curr_buffer->set_tensor_table(std::move(curr_table));
curr_buffer->set_column_name_map(col_name_id_map_); curr_buffer->set_column_name_map(col_name_id_map_);
MS_LOG(DEBUG) << "Zip operator finished one buffer, pushing, rows " << curr_buffer->NumRows() << ", cols " MS_LOG(DEBUG) << "Zip operator finished one buffer, pushing, rows " << curr_buffer->NumRows() << ", cols "
@@ -105,15 +104,14 @@ Status ZipOp::operator()() {
MS_LOG(DEBUG) << "Zip operator is now draining child inputs."; MS_LOG(DEBUG) << "Zip operator is now draining child inputs.";
RETURN_IF_NOT_OK(drainPipeline()); RETURN_IF_NOT_OK(drainPipeline());
// Now that we have drained child inputs, send the eoe up. // Now that we have drained child inputs, send the eoe up.
RETURN_IF_NOT_OK(
out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
} }
} }


// 5 handle eof // 5 handle eof
// propagate eof here. // propagate eof here.
MS_LOG(INFO) << "Zip operator got EOF, propagating."; MS_LOG(INFO) << "Zip operator got EOF, propagating.";
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
return Status::OK(); return Status::OK();
} }




+ 1
- 1
mindspore/ccsrc/dataset/engine/db_connector.h View File

@@ -65,7 +65,7 @@ class DbConnector : public Connector<std::unique_ptr<DataBuffer>> {
RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; }));
// Once an EOF message is encountered this flag will be set and we can return early. // Once an EOF message is encountered this flag will be set and we can return early.
if (end_of_file_) { if (end_of_file_) {
*result = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
*result = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
} else { } else {
RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result));
if (*result == nullptr) { if (*result == nullptr) {


+ 1
- 1
mindspore/ccsrc/dataset/engine/execution_tree.cc View File

@@ -24,7 +24,7 @@ namespace mindspore {
namespace dataset { namespace dataset {
// Constructor // Constructor
ExecutionTree::ExecutionTree() : id_count_(0) { ExecutionTree::ExecutionTree() : id_count_(0) {
tg_ = mindspore::make_unique<TaskGroup>();
tg_ = std::make_unique<TaskGroup>();
tree_state_ = kDeTStateInit; tree_state_ = kDeTStateInit;
prepare_flags_ = kDePrepNone; prepare_flags_ = kDePrepNone;
} }


+ 1
- 2
mindspore/ccsrc/dataset/kernels/image/image_utils.cc View File

@@ -24,7 +24,6 @@
#include "dataset/core/cv_tensor.h" #include "dataset/core/cv_tensor.h"
#include "dataset/core/tensor.h" #include "dataset/core/tensor.h"
#include "dataset/core/tensor_shape.h" #include "dataset/core/tensor_shape.h"
#include "dataset/util/make_unique.h"
#include "dataset/util/random.h" #include "dataset/util/random.h"


#define MAX_INT_PRECISION 16777216 // float int precision is 16777216 #define MAX_INT_PRECISION 16777216 // float int precision is 16777216
@@ -376,7 +375,7 @@ Status HwcToChw(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output)
int width = input_cv->shape()[1]; int width = input_cv->shape()[1];
int num_channels = input_cv->shape()[2]; int num_channels = input_cv->shape()[2];


auto output_cv = mindspore::make_unique<CVTensor>(TensorShape{num_channels, height, width}, input_cv->type());
auto output_cv = std::make_unique<CVTensor>(TensorShape{num_channels, height, width}, input_cv->type());
for (int i = 0; i < num_channels; ++i) { for (int i = 0; i < num_channels; ++i) {
cv::Mat mat; cv::Mat mat;
RETURN_IF_NOT_OK(output_cv->Mat({i}, &mat)); RETURN_IF_NOT_OK(output_cv->Mat({i}, &mat));


+ 0
- 1
mindspore/ccsrc/dataset/kernels/py_func_op.cc View File

@@ -20,7 +20,6 @@


#include "dataset/core/tensor.h" #include "dataset/core/tensor.h"
#include "dataset/kernels/tensor_op.h" #include "dataset/kernels/tensor_op.h"
#include "dataset/util/make_unique.h"
#include "dataset/util/status.h" #include "dataset/util/status.h"


namespace mindspore { namespace mindspore {


+ 0
- 1
mindspore/ccsrc/dataset/util/arena.cc View File

@@ -16,7 +16,6 @@
#include "dataset/util/arena.h" #include "dataset/util/arena.h"
#include <unistd.h> #include <unistd.h>
#include <utility> #include <utility>
#include "dataset/util/make_unique.h"
#include "dataset/util/system_pool.h" #include "dataset/util/system_pool.h"
#include "dataset/util/de_error.h" #include "dataset/util/de_error.h"
#include "./securec.h" #include "./securec.h"


+ 1
- 3
mindspore/ccsrc/dataset/util/circular_pool.cc View File

@@ -18,10 +18,8 @@
#include <algorithm> #include <algorithm>
#include <limits> #include <limits>
#include <utility> #include <utility>

#include "./securec.h" #include "./securec.h"

#include "dataset/util/make_unique.h"
#include "dataset/util/de_error.h"
#include "dataset/util/system_pool.h" #include "dataset/util/system_pool.h"


namespace mindspore { namespace mindspore {


+ 7
- 0
mindspore/ccsrc/dataset/util/de_error.h View File

@@ -16,6 +16,13 @@
#ifndef DATASET_UTIL_DE_ERROR_H_ #ifndef DATASET_UTIL_DE_ERROR_H_
#define DATASET_UTIL_DE_ERROR_H_ #define DATASET_UTIL_DE_ERROR_H_


#ifdef DEBUG
#include <cassert>
#define DS_ASSERT(f) assert(f)
#else
#define DS_ASSERT(f) ((void)0)
#endif

#include <map> #include <map>
#include "utils/error_code.h" #include "utils/error_code.h"




+ 1
- 2
mindspore/ccsrc/dataset/util/list.h View File

@@ -18,8 +18,7 @@


#include <iostream> #include <iostream>
#include <iterator> #include <iterator>

#include "dataset/util/make_unique.h"
#include "dataset/util/de_error.h"


namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {


+ 1
- 0
mindspore/ccsrc/dataset/util/lock.cc View File

@@ -14,6 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
#include "dataset/util/lock.h" #include "dataset/util/lock.h"
#include "dataset/util/de_error.h"


namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {


+ 0
- 1
mindspore/ccsrc/dataset/util/lock.h View File

@@ -19,7 +19,6 @@
#include <atomic> #include <atomic>
#include <condition_variable> #include <condition_variable>
#include <mutex> #include <mutex>
#include "dataset/util/make_unique.h"


namespace mindspore { namespace mindspore {
namespace dataset { namespace dataset {


+ 0
- 37
mindspore/ccsrc/dataset/util/make_unique.h View File

@@ -1,37 +0,0 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef DATASET_UTIL_MAKE_UNIQUE_H_
#define DATASET_UTIL_MAKE_UNIQUE_H_

#ifdef DEBUG
#include <cassert>
#define DS_ASSERT(f) assert(f)
#else
#define DS_ASSERT(f) ((void)0)
#endif

#include <memory>
#include <type_traits>
#include <utility>
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"

namespace mindspore {
using std::make_unique;
} // namespace mindspore

#endif // DATASET_UTIL_MAKE_UNIQUE_H_

+ 1
- 1
mindspore/ccsrc/dataset/util/queue.h View File

@@ -212,7 +212,7 @@ class QueueList {
void Init(int num_queues, int capacity) { void Init(int num_queues, int capacity) {
queue_list_.reserve(num_queues); queue_list_.reserve(num_queues);
for (int i = 0; i < num_queues; i++) { for (int i = 0; i < num_queues; i++) {
queue_list_.emplace_back(mindspore::make_unique<Queue<T>>(capacity));
queue_list_.emplace_back(std::make_unique<Queue<T>>(capacity));
} }
} }




+ 0
- 1
mindspore/ccsrc/dataset/util/task.h View File

@@ -27,7 +27,6 @@
#include <string> #include <string>
#include <thread> #include <thread>
#include "dataset/util/de_error.h" #include "dataset/util/de_error.h"
#include "dataset/util/make_unique.h"
#include "dataset/util/intrp_resource.h" #include "dataset/util/intrp_resource.h"
#include "dataset/util/list.h" #include "dataset/util/list.h"
#include "dataset/util/memory_pool.h" #include "dataset/util/memory_pool.h"


+ 1
- 2
mindspore/ccsrc/device/gpu/blocking_queue.cc View File

@@ -17,7 +17,6 @@
#include "device/gpu/blocking_queue.h" #include "device/gpu/blocking_queue.h"
#include <chrono> #include <chrono>
#include "device/gpu/gpu_common.h" #include "device/gpu/gpu_common.h"
#include "dataset/util/make_unique.h"
#include "common/utils.h" #include "common/utils.h"


namespace mindspore { namespace mindspore {
@@ -32,7 +31,7 @@ GpuQueue::GpuQueue(void *addr, size_t feature_size, size_t label_size, size_t ca
stream_(0), stream_(0),
node_info_(nullptr) { node_info_(nullptr) {
CHECK_CUDA_RET_WITH_ERROR(cudaStreamCreate(&stream_), "Cuda Create Stream Failed"); CHECK_CUDA_RET_WITH_ERROR(cudaStreamCreate(&stream_), "Cuda Create Stream Failed");
node_info_ = mindspore::make_unique<NodeInfo[]>(capacity);
node_info_ = std::make_unique<NodeInfo[]>(capacity);
} }


GpuQueue::~GpuQueue() { buffer_ = nullptr; } GpuQueue::~GpuQueue() { buffer_ = nullptr; }


+ 2
- 3
mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.h View File

@@ -23,7 +23,6 @@
#include <vector> #include <vector>
#include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel.h"
#include "kernel/gpu/gpu_kernel_factory.h" #include "kernel/gpu/gpu_kernel_factory.h"
#include "dataset/util/make_unique.h"
#include "kernel/gpu/kernel_constants.h" #include "kernel/gpu/kernel_constants.h"


namespace mindspore { namespace mindspore {
@@ -74,8 +73,8 @@ class BiasAddGpuKernel : public GpuKernel {


// Expand to 4 dims for cudnnSetTensorNdDescriptorEx. // Expand to 4 dims for cudnnSetTensorNdDescriptorEx.
auto cudnn_dims = std::max(num_dims, 4UL); auto cudnn_dims = std::max(num_dims, 4UL);
std::unique_ptr<int[]> x_dims = mindspore::make_unique<int[]>(cudnn_dims);
std::unique_ptr<int[]> b_dims = mindspore::make_unique<int[]>(cudnn_dims);
std::unique_ptr<int[]> x_dims = std::make_unique<int[]>(cudnn_dims);
std::unique_ptr<int[]> b_dims = std::make_unique<int[]>(cudnn_dims);
for (size_t i = 0; i < cudnn_dims; i++) { for (size_t i = 0; i < cudnn_dims; i++) {
x_dims[i] = (i < num_dims) ? SizeToInt(x_shape[i]) : 1; x_dims[i] = (i < num_dims) ? SizeToInt(x_shape[i]) : 1;
b_dims[i] = (i == pos) ? SizeToInt(x_shape[i]) : 1; b_dims[i] = (i == pos) ? SizeToInt(x_shape[i]) : 1;


+ 2
- 3
mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.h View File

@@ -26,7 +26,6 @@
#include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel.h"
#include "kernel/gpu/gpu_kernel_factory.h" #include "kernel/gpu/gpu_kernel_factory.h"
#include "kernel/gpu/kernel_constants.h" #include "kernel/gpu/kernel_constants.h"
#include "dataset/util/make_unique.h"


namespace mindspore { namespace mindspore {
namespace kernel { namespace kernel {
@@ -84,8 +83,8 @@ class BiasAddGradGpuKernel : public GpuKernel {


// Expand to 4 dims for cudnnSetTensorNdDescriptorEx. // Expand to 4 dims for cudnnSetTensorNdDescriptorEx.
auto cudnn_dims = std::max(num_dims, 4UL); auto cudnn_dims = std::max(num_dims, 4UL);
std::unique_ptr<int[]> dy_dims = mindspore::make_unique<int[]>(cudnn_dims);
std::unique_ptr<int[]> db_dims = mindspore::make_unique<int[]>(cudnn_dims);
std::unique_ptr<int[]> dy_dims = std::make_unique<int[]>(cudnn_dims);
std::unique_ptr<int[]> db_dims = std::make_unique<int[]>(cudnn_dims);
for (size_t i = 0; i < cudnn_dims; i++) { for (size_t i = 0; i < cudnn_dims; i++) {
dy_dims[i] = (i < num_dims) ? SizeToInt(dy_shape[i]) : 1; dy_dims[i] = (i < num_dims) ? SizeToInt(dy_shape[i]) : 1;
db_dims[i] = (i == pos) ? SizeToInt(dy_shape[i]) : 1; db_dims[i] = (i == pos) ? SizeToInt(dy_shape[i]) : 1;


+ 2
- 3
mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.h View File

@@ -22,7 +22,6 @@
#include <memory> #include <memory>
#include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel.h"
#include "kernel/gpu/gpu_kernel_factory.h" #include "kernel/gpu/gpu_kernel_factory.h"
#include "dataset/util/make_unique.h"
#include "kernel/gpu/kernel_constants.h" #include "kernel/gpu/kernel_constants.h"


namespace mindspore { namespace mindspore {
@@ -144,8 +143,8 @@ class LstmGpuKernel : public GpuKernel {
int x_dims[3]{batch_size_, input_size_, 1}; int x_dims[3]{batch_size_, input_size_, 1};
int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1};


x_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
x_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);


for (size_t i = 0; i < IntToSize(seq_len_); ++i) { for (size_t i = 0; i < IntToSize(seq_len_); ++i) {
CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed");


+ 3
- 4
mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.h View File

@@ -23,7 +23,6 @@
#include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel.h"
#include "kernel/gpu/gpu_kernel_factory.h" #include "kernel/gpu/gpu_kernel_factory.h"
#include "kernel/gpu/kernel_constants.h" #include "kernel/gpu/kernel_constants.h"
#include "dataset/util/make_unique.h"


namespace mindspore { namespace mindspore {
namespace kernel { namespace kernel {
@@ -212,9 +211,9 @@ class LstmGradDataGpuKernel : public GpuKernel {
int x_dims[3]{batch_size_, input_size_, 1}; int x_dims[3]{batch_size_, input_size_, 1};
int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1};


dx_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
dy_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
dx_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
dy_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);


for (size_t i = 0; i < IntToSize(seq_len_); ++i) { for (size_t i = 0; i < IntToSize(seq_len_); ++i) {
CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_[i]), "create x_desc failed"); CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_[i]), "create x_desc failed");


+ 2
- 3
mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.h View File

@@ -22,7 +22,6 @@
#include <memory> #include <memory>
#include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel.h"
#include "kernel/gpu/gpu_kernel_factory.h" #include "kernel/gpu/gpu_kernel_factory.h"
#include "dataset/util/make_unique.h"
#include "kernel/gpu/kernel_constants.h" #include "kernel/gpu/kernel_constants.h"
namespace mindspore { namespace mindspore {
namespace kernel { namespace kernel {
@@ -169,8 +168,8 @@ class LstmGradWeightGpuKernel : public GpuKernel {
int x_dims[3]{batch_size_, input_size_, 1}; int x_dims[3]{batch_size_, input_size_, 1};
int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1};


x_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
x_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);


for (size_t i = 0; i < IntToSize(seq_len_); ++i) { for (size_t i = 0; i < IntToSize(seq_len_); ++i) {
CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed");


+ 1
- 1
tests/ut/cpp/dataset/celeba_op_test.cc View File

@@ -116,7 +116,7 @@ TEST_F(MindDataTestCelebaDataset, TestCelebaRepeat) {


TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) { TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) {
std::vector<int64_t> indices({1}); std::vector<int64_t> indices({1});
std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices);
std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices);
uint32_t expect_labels[1][40] = {{0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}}; uint32_t expect_labels[1][40] = {{0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}};
std::string dir = datasets_root_path_ + "/testCelebAData/"; std::string dir = datasets_root_path_ + "/testCelebAData/";
uint32_t count = 0; uint32_t count = 0;


+ 1
- 1
tests/ut/cpp/dataset/cifar_op_test.cc View File

@@ -92,7 +92,7 @@ TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar10) {
TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) { TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) {
uint32_t original_seed = GlobalContext::config_manager()->seed(); uint32_t original_seed = GlobalContext::config_manager()->seed();
GlobalContext::config_manager()->set_seed(0); GlobalContext::config_manager()->set_seed(0);
std::unique_ptr<Sampler> sampler = mindspore::make_unique<RandomSampler>(true, 12);
std::unique_ptr<Sampler> sampler = std::make_unique<RandomSampler>(true, 12);
std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler), 100)}); auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler), 100)});
tree->Prepare(); tree->Prepare();


+ 7
- 7
tests/ut/cpp/dataset/image_folder_op_test.cc View File

@@ -138,7 +138,7 @@ TEST_F(MindDataTestImageFolderSampler, TestRandomImageFolder) {
TEST_F(MindDataTestImageFolderSampler, TestRandomSamplerImageFolder) { TEST_F(MindDataTestImageFolderSampler, TestRandomSamplerImageFolder) {
int32_t original_seed = GlobalContext::config_manager()->seed(); int32_t original_seed = GlobalContext::config_manager()->seed();
GlobalContext::config_manager()->set_seed(0); GlobalContext::config_manager()->set_seed(0);
std::unique_ptr<Sampler> sampler = mindspore::make_unique<RandomSampler>(true, 12);
std::unique_ptr<Sampler> sampler = std::make_unique<RandomSampler>(true, 12);
int32_t res[] = {2, 2, 2, 3, 2, 3, 2, 3, 1, 2, 2, 1}; // ground truth label int32_t res[] = {2, 2, 2, 3, 2, 3, 2, 3, 1, 2, 2, 1}; // ground truth label
std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))});
@@ -200,7 +200,7 @@ TEST_F(MindDataTestImageFolderSampler, TestSequentialImageFolderWithRepeatBatch)
TEST_F(MindDataTestImageFolderSampler, TestSubsetRandomSamplerImageFolder) { TEST_F(MindDataTestImageFolderSampler, TestSubsetRandomSamplerImageFolder) {
// id range 0 - 10 is label 0, and id range 11 - 21 is label 1 // id range 0 - 10 is label 0, and id range 11 - 21 is label 1
std::vector<int64_t> indices({0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11}); std::vector<int64_t> indices({0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11});
std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices);
std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices);
std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
// Expect 6 samples for label 0 and 1 // Expect 6 samples for label 0 and 1
int res[2] = {6, 6}; int res[2] = {6, 6};
@@ -238,7 +238,7 @@ TEST_F(MindDataTestImageFolderSampler, TestWeightedRandomSamplerImageFolder) {


// create sampler with replacement = replacement // create sampler with replacement = replacement
std::unique_ptr<Sampler> sampler = std::unique_ptr<Sampler> sampler =
mindspore::make_unique<WeightedRandomSampler>(weights, num_samples, true, samples_per_buffer);
std::make_unique<WeightedRandomSampler>(weights, num_samples, true, samples_per_buffer);


std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))});
@@ -295,7 +295,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderClassIndex) {
} }


TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) { TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) {
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(11, 10, false);
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(11, 10, false);
std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler)), Repeat(4)}); auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler)), Repeat(4)});
tree->Prepare(); tree->Prepare();
@@ -322,7 +322,7 @@ TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) {
} }


TEST_F(MindDataTestImageFolderSampler, TestPKSamplerImageFolder) { TEST_F(MindDataTestImageFolderSampler, TestPKSamplerImageFolder) {
std::unique_ptr<Sampler> sampler = mindspore::make_unique<PKSampler>(3, false, 4);
std::unique_ptr<Sampler> sampler = std::make_unique<PKSampler>(3, false, 4);
int32_t res[] = {0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; // ground truth label int32_t res[] = {0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; // ground truth label
std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))});
@@ -431,7 +431,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderDatasetSize) {
} }


TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) { TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) {
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(4, 0, false);
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(4, 0, false);
std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
// numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler), {}, 5)}); auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler), {}, 5)});
@@ -460,7 +460,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) {
} }


TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding2) { TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding2) {
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(4, 3, false);
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(4, 3, false);
std::string folder_path = datasets_root_path_ + "/testPK/data"; std::string folder_path = datasets_root_path_ + "/testPK/data";
// numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode
auto tree = Build({ImageFolder(16, 16, 32, folder_path, false, std::move(sampler), {}, 12)}); auto tree = Build({ImageFolder(16, 16, 32, folder_path, false, std::move(sampler), {}, 12)});


+ 1
- 1
tests/ut/cpp/dataset/manifest_op_test.cc View File

@@ -86,7 +86,7 @@ TEST_F(MindDataTestManifest, TestSequentialManifestWithRepeat) {


TEST_F(MindDataTestManifest, TestSubsetRandomSamplerManifest) { TEST_F(MindDataTestManifest, TestSubsetRandomSamplerManifest) {
std::vector<int64_t> indices({1}); std::vector<int64_t> indices({1});
std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices);
std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices);
std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; std::string file = datasets_root_path_ + "/testManifestData/cpp.json";
// Expect 6 samples for label 0 and 1 // Expect 6 samples for label 0 and 1
auto tree = Build({Manifest(16, 2, 32, file, "train", std::move(sampler))}); auto tree = Build({Manifest(16, 2, 32, file, "train", std::move(sampler))});


+ 1
- 1
tests/ut/cpp/dataset/project_op_test.cc View File

@@ -45,7 +45,7 @@ TEST_F(MindDataTestProjectOp, TestProjectProject) {
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16) .SetWorkerConnectorSize(16)
.SetNumWorkers(16); .SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk()); Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk());


+ 1
- 1
tests/ut/cpp/dataset/stand_alone_samplers_test.cc View File

@@ -74,7 +74,7 @@ TEST_F(MindDataTestStandAloneSampler, TestDistributedSampler) {
std::unique_ptr<DataBuffer> db; std::unique_ptr<DataBuffer> db;
std::shared_ptr<Tensor> tensor; std::shared_ptr<Tensor> tensor;
for (int i = 0; i < 6; i++) { for (int i = 0; i < 6; i++) {
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(3, i % 3, (i < 3 ? false : true));
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(3, i % 3, (i < 3 ? false : true));
sampler->Init(&mock); sampler->Init(&mock);
sampler->GetNextBuffer(&db); sampler->GetNextBuffer(&db);
db->GetTensor(&tensor, 0, 0); db->GetTensor(&tensor, 0, 0);


+ 10
- 10
tests/ut/cpp/dataset/tfReader_op_test.cc View File

@@ -48,7 +48,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasic1) {
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path})
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetNumWorkers(16); .SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
@@ -102,7 +102,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeRowsPerBuffer) {
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path})
.SetRowsPerBuffer(500) .SetRowsPerBuffer(500)
.SetNumWorkers(16); .SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
@@ -156,7 +156,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSmallRowsPerBuffer) {
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path})
.SetRowsPerBuffer(1) .SetRowsPerBuffer(1)
.SetNumWorkers(16); .SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
@@ -211,7 +211,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeQueueSize) {
.SetWorkerConnectorSize(1) .SetWorkerConnectorSize(1)
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetNumWorkers(16); .SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
@@ -265,7 +265,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderOneThread) {
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path})
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetNumWorkers(1); .SetNumWorkers(1);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
@@ -321,7 +321,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderRepeat) {
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16) .SetWorkerConnectorSize(16)
.SetNumWorkers(16); .SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
Status rc= builder.Build(&my_tfreader_op); Status rc= builder.Build(&my_tfreader_op);
@@ -379,7 +379,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSchemaConstructor) {
std::string dataset_path; std::string dataset_path;
dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; dataset_path = datasets_root_path_ + "/testTFTestAllTypes";


std::unique_ptr<DataSchema> data_schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> data_schema = std::make_unique<DataSchema>();
std::vector<std::string> columns_to_load; std::vector<std::string> columns_to_load;
columns_to_load.push_back("col_sint32"); columns_to_load.push_back("col_sint32");
columns_to_load.push_back("col_binary"); columns_to_load.push_back("col_binary");
@@ -445,7 +445,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Row) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));


@@ -503,7 +503,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Buffer) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));


@@ -561,7 +561,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake7Rows) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));




Loading…
Cancel
Save