| @@ -23,7 +23,6 @@ | |||||
| #include "dataset/engine/datasetops/source/image_folder_op.h" | #include "dataset/engine/datasetops/source/image_folder_op.h" | ||||
| #include "dataset/engine/datasetops/source/mnist_op.h" | #include "dataset/engine/datasetops/source/mnist_op.h" | ||||
| #include "dataset/engine/datasetops/source/voc_op.h" | #include "dataset/engine/datasetops/source/voc_op.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/core/tensor.h" | #include "dataset/core/tensor.h" | ||||
| #include "dataset/engine/dataset_iterator.h" | #include "dataset/engine/dataset_iterator.h" | ||||
| #include "dataset/engine/datasetops/source/manifest_op.h" | #include "dataset/engine/datasetops/source/manifest_op.h" | ||||
| @@ -123,7 +122,7 @@ Status DEPipeline::AssignRootNode(const DsOpPtr &dataset_op) { return (tree_->As | |||||
| Status DEPipeline::LaunchTreeExec() { | Status DEPipeline::LaunchTreeExec() { | ||||
| RETURN_IF_NOT_OK(tree_->Prepare()); | RETURN_IF_NOT_OK(tree_->Prepare()); | ||||
| RETURN_IF_NOT_OK(tree_->Launch()); | RETURN_IF_NOT_OK(tree_->Launch()); | ||||
| iterator_ = make_unique<DatasetIterator>(tree_); | |||||
| iterator_ = std::make_unique<DatasetIterator>(tree_); | |||||
| if (iterator_ == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create an Iterator."); | if (iterator_ == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create an Iterator."); | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -311,7 +310,7 @@ Status DEPipeline::ParseStorageOp(const py::dict &args, std::shared_ptr<DatasetO | |||||
| if (!args["schema"].is_none()) { | if (!args["schema"].is_none()) { | ||||
| (void)builder->SetSchemaFile(ToString(args["schema"])); | (void)builder->SetSchemaFile(ToString(args["schema"])); | ||||
| } else if (!args["schema_json_string"].is_none()) { | } else if (!args["schema_json_string"].is_none()) { | ||||
| std::unique_ptr<DataSchema> schema = make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| std::string s = ToString(args["schema_json_string"]); | std::string s = ToString(args["schema_json_string"]); | ||||
| RETURN_IF_NOT_OK(schema->LoadSchemaString(s, std::vector<std::string>())); | RETURN_IF_NOT_OK(schema->LoadSchemaString(s, std::vector<std::string>())); | ||||
| (void)builder->SetNumRows(schema->num_rows()); | (void)builder->SetNumRows(schema->num_rows()); | ||||
| @@ -689,7 +688,7 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr<Dataset | |||||
| } | } | ||||
| } | } | ||||
| if (schema_exists) { | if (schema_exists) { | ||||
| std::unique_ptr<DataSchema> schema = make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| if (args.contains("schema_file_path")) { | if (args.contains("schema_file_path")) { | ||||
| RETURN_IF_NOT_OK(schema->LoadSchemaFile(ToString(args["schema_file_path"]), columns_to_load)); | RETURN_IF_NOT_OK(schema->LoadSchemaFile(ToString(args["schema_file_path"]), columns_to_load)); | ||||
| } else { | } else { | ||||
| @@ -55,9 +55,9 @@ Status GlobalContext::Init() { | |||||
| // For testing we can use Dummy pool instead | // For testing we can use Dummy pool instead | ||||
| // Create some tensor allocators for the different types and hook them into the pool. | // Create some tensor allocators for the different types and hook them into the pool. | ||||
| tensor_allocator_ = mindspore::make_unique<Allocator<Tensor>>(mem_pool_); | |||||
| cv_tensor_allocator_ = mindspore::make_unique<Allocator<CVTensor>>(mem_pool_); | |||||
| int_allocator_ = mindspore::make_unique<IntAlloc>(mem_pool_); | |||||
| tensor_allocator_ = std::make_unique<Allocator<Tensor>>(mem_pool_); | |||||
| cv_tensor_allocator_ = std::make_unique<Allocator<CVTensor>>(mem_pool_); | |||||
| int_allocator_ = std::make_unique<IntAlloc>(mem_pool_); | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -28,7 +28,6 @@ | |||||
| #include "dataset/core/global_context.h" | #include "dataset/core/global_context.h" | ||||
| #include "dataset/core/pybind_support.h" | #include "dataset/core/pybind_support.h" | ||||
| #include "dataset/core/tensor_shape.h" | #include "dataset/core/tensor_shape.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| namespace py = pybind11; | namespace py = pybind11; | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -53,7 +52,7 @@ namespace dataset { | |||||
| Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) { | Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) { | ||||
| // grab the mem pool from global context and create the allocator for char data area | // grab the mem pool from global context and create the allocator for char data area | ||||
| std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool(); | std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool(); | ||||
| data_allocator_ = mindspore::make_unique<Allocator<unsigned char>>(global_pool); | |||||
| data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool); | |||||
| } | } | ||||
| Tensor::Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data) : Tensor(shape, type) { | Tensor::Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data) : Tensor(shape, type) { | ||||
| @@ -137,7 +136,7 @@ Status Tensor::CreateTensor(std::shared_ptr<Tensor> *ptr, py::array arr) { | |||||
| if ((*ptr)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type."); | if ((*ptr)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type."); | ||||
| std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool(); | std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool(); | ||||
| (*ptr)->data_allocator_ = mindspore::make_unique<Allocator<unsigned char>>(global_pool); | |||||
| (*ptr)->data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool); | |||||
| static_cast<void>((*ptr)->StartAddr()); | static_cast<void>((*ptr)->StartAddr()); | ||||
| int64_t byte_size = (*ptr)->SizeInBytes(); | int64_t byte_size = (*ptr)->SizeInBytes(); | ||||
| unsigned char *data = static_cast<unsigned char *>(arr.request().ptr); | unsigned char *data = static_cast<unsigned char *>(arr.request().ptr); | ||||
| @@ -40,7 +40,7 @@ Status DataBuffer::CreateDataBuffer( | |||||
| case DatasetType::kTf: { | case DatasetType::kTf: { | ||||
| // This type of buffer is for TF record data. | // This type of buffer is for TF record data. | ||||
| // Allocate derived class version for a TF buffers | // Allocate derived class version for a TF buffers | ||||
| new_data_buffer = mindspore::make_unique<TFBuffer>(id, kDeBFlagNone, storage_client); | |||||
| new_data_buffer = std::make_unique<TFBuffer>(id, kDeBFlagNone, storage_client); | |||||
| break; | break; | ||||
| } | } | ||||
| default: { | default: { | ||||
| @@ -26,8 +26,8 @@ | |||||
| #include "common/utils.h" | #include "common/utils.h" | ||||
| #include "dataset/util/status.h" | #include "dataset/util/status.h" | ||||
| #include "dataset/core/tensor_shape.h" | #include "dataset/core/tensor_shape.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #include "dataset/util/de_error.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -58,7 +58,7 @@ ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, Ten | |||||
| // our shape. Otherwise, set our shape to be empty. | // our shape. Otherwise, set our shape to be empty. | ||||
| if (in_shape != nullptr) { | if (in_shape != nullptr) { | ||||
| // Create a shape and copy construct it into our column's shape. | // Create a shape and copy construct it into our column's shape. | ||||
| tensor_shape_ = mindspore::make_unique<TensorShape>(*in_shape); | |||||
| tensor_shape_ = std::make_unique<TensorShape>(*in_shape); | |||||
| } else { | } else { | ||||
| tensor_shape_ = nullptr; | tensor_shape_ = nullptr; | ||||
| } | } | ||||
| @@ -75,7 +75,7 @@ ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, Ten | |||||
| ColDescriptor::ColDescriptor(const ColDescriptor &in_cd) | ColDescriptor::ColDescriptor(const ColDescriptor &in_cd) | ||||
| : type_(in_cd.type_), rank_(in_cd.rank_), tensor_impl_(in_cd.tensor_impl_), col_name_(in_cd.col_name_) { | : type_(in_cd.type_), rank_(in_cd.rank_), tensor_impl_(in_cd.tensor_impl_), col_name_(in_cd.col_name_) { | ||||
| // If it has a tensor shape, make a copy of it with our own unique_ptr. | // If it has a tensor shape, make a copy of it with our own unique_ptr. | ||||
| tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique<TensorShape>(in_cd.shape()) : nullptr; | |||||
| tensor_shape_ = in_cd.hasShape() ? std::make_unique<TensorShape>(in_cd.shape()) : nullptr; | |||||
| } | } | ||||
| // Assignment overload | // Assignment overload | ||||
| @@ -86,7 +86,7 @@ ColDescriptor &ColDescriptor::operator=(const ColDescriptor &in_cd) { | |||||
| tensor_impl_ = in_cd.tensor_impl_; | tensor_impl_ = in_cd.tensor_impl_; | ||||
| col_name_ = in_cd.col_name_; | col_name_ = in_cd.col_name_; | ||||
| // If it has a tensor shape, make a copy of it with our own unique_ptr. | // If it has a tensor shape, make a copy of it with our own unique_ptr. | ||||
| tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique<TensorShape>(in_cd.shape()) : nullptr; | |||||
| tensor_shape_ = in_cd.hasShape() ? std::make_unique<TensorShape>(in_cd.shape()) : nullptr; | |||||
| } | } | ||||
| return *this; | return *this; | ||||
| } | } | ||||
| @@ -59,8 +59,8 @@ Status BatchOp::operator()() { | |||||
| TaskManager::FindMe()->Post(); | TaskManager::FindMe()->Post(); | ||||
| int32_t epoch_num = 0, batch_num = 0, cnt = 0; | int32_t epoch_num = 0, batch_num = 0, cnt = 0; | ||||
| TensorRow new_row; | TensorRow new_row; | ||||
| std::unique_ptr<TensorQTable> table = make_unique<TensorQTable>(); | |||||
| child_iterator_ = mindspore::make_unique<ChildIterator>(this, 0, 0); | |||||
| std::unique_ptr<TensorQTable> table = std::make_unique<TensorQTable>(); | |||||
| child_iterator_ = std::make_unique<ChildIterator>(this, 0, 0); | |||||
| RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); | RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); | ||||
| column_name_map_ = child_iterator_->col_name_id_map(); | column_name_map_ = child_iterator_->col_name_id_map(); | ||||
| int32_t cur_batch_size = 0; | int32_t cur_batch_size = 0; | ||||
| @@ -72,7 +72,7 @@ Status BatchOp::operator()() { | |||||
| if (table->size() == static_cast<size_t>(cur_batch_size)) { | if (table->size() == static_cast<size_t>(cur_batch_size)) { | ||||
| RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( | RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( | ||||
| std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); | std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); | ||||
| table = make_unique<TensorQTable>(); | |||||
| table = std::make_unique<TensorQTable>(); | |||||
| RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num))); | RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num))); | ||||
| } | } | ||||
| RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); | RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); | ||||
| @@ -82,7 +82,7 @@ Status BatchOp::operator()() { | |||||
| RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( | RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( | ||||
| std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); | std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); | ||||
| } | } | ||||
| table = make_unique<TensorQTable>(); // this drops when drop == true | |||||
| table = std::make_unique<TensorQTable>(); // this drops when drop == true | |||||
| // end of the current epoch, batch_num should start from 0 again | // end of the current epoch, batch_num should start from 0 again | ||||
| batch_num = 0; | batch_num = 0; | ||||
| epoch_num++; | epoch_num++; | ||||
| @@ -153,9 +153,9 @@ Status BatchOp::WorkerEntry(int32_t workerId) { | |||||
| RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair)); | RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair)); | ||||
| while (table_pair.second.ctrl_ != batchCtrl::kQuit) { | while (table_pair.second.ctrl_ != batchCtrl::kQuit) { | ||||
| if (table_pair.second.ctrl_ == batchCtrl::kEOE) { | if (table_pair.second.ctrl_ == batchCtrl::kEOE) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| } else if (table_pair.second.ctrl_ == batchCtrl::kEOF) { | } else if (table_pair.second.ctrl_ == batchCtrl::kEOF) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| } else if (table_pair.second.ctrl_ == batchCtrl::kNoCtrl) { | } else if (table_pair.second.ctrl_ == batchCtrl::kNoCtrl) { | ||||
| std::unique_ptr<DataBuffer> db = nullptr; | std::unique_ptr<DataBuffer> db = nullptr; | ||||
| RETURN_IF_NOT_OK(MakeBatchedBuffer(std::move(table_pair), &db)); | RETURN_IF_NOT_OK(MakeBatchedBuffer(std::move(table_pair), &db)); | ||||
| @@ -170,8 +170,8 @@ Status BatchOp::MakeBatchedBuffer(std::pair<std::unique_ptr<TensorQTable>, CBatc | |||||
| std::unique_ptr<DataBuffer> *db) { | std::unique_ptr<DataBuffer> *db) { | ||||
| RETURN_UNEXPECTED_IF_NULL(table_pair.first); | RETURN_UNEXPECTED_IF_NULL(table_pair.first); | ||||
| if (!input_column_names_.empty()) RETURN_IF_NOT_OK(MapColumns(&table_pair)); // pass it through pyfunc | if (!input_column_names_.empty()) RETURN_IF_NOT_OK(MapColumns(&table_pair)); // pass it through pyfunc | ||||
| (*db) = make_unique<DataBuffer>(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<TensorQTable> dest_table = make_unique<TensorQTable>(); | |||||
| (*db) = std::make_unique<DataBuffer>(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<TensorQTable> dest_table = std::make_unique<TensorQTable>(); | |||||
| RETURN_IF_NOT_OK(BatchRows(&table_pair.first, &dest_table, table_pair.first->size())); | RETURN_IF_NOT_OK(BatchRows(&table_pair.first, &dest_table, table_pair.first->size())); | ||||
| (*db)->set_tensor_table(std::move(dest_table)); | (*db)->set_tensor_table(std::move(dest_table)); | ||||
| (*db)->set_column_name_map(column_name_map_); | (*db)->set_column_name_map(column_name_map_); | ||||
| @@ -80,9 +80,9 @@ void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) { | |||||
| MS_LOG(INFO) << "Creating connector in tree operator: " << operator_id_ << ". Producer: " << num_producers | MS_LOG(INFO) << "Creating connector in tree operator: " << operator_id_ << ". Producer: " << num_producers | ||||
| << ". Consumer: " << num_consumers << "."; | << ". Consumer: " << num_consumers << "."; | ||||
| if (oc_queue_size_ > 0) { | if (oc_queue_size_ > 0) { | ||||
| out_connector_ = mindspore::make_unique<DbConnector>(num_producers, // The number of producers | |||||
| num_consumers, // Only one consumer (the training App) | |||||
| oc_queue_size_); | |||||
| out_connector_ = std::make_unique<DbConnector>(num_producers, // The number of producers | |||||
| num_consumers, // Only one consumer (the training App) | |||||
| oc_queue_size_); | |||||
| } else { | } else { | ||||
| // Some op's may choose not to have an output connector | // Some op's may choose not to have an output connector | ||||
| MS_LOG(INFO) << "Bypassed connector creation for tree operator: " << operator_id_ << "."; | MS_LOG(INFO) << "Bypassed connector creation for tree operator: " << operator_id_ << "."; | ||||
| @@ -149,7 +149,7 @@ Status DatasetOp::GetNextInput(std::unique_ptr<DataBuffer> *p_buffer, int32_t wo | |||||
| // The base class implementation simply flows the eoe message to output. Derived classes | // The base class implementation simply flows the eoe message to output. Derived classes | ||||
| // may override if they need to perform special eoe handling. | // may override if they need to perform special eoe handling. | ||||
| Status DatasetOp::EoeReceived(int32_t worker_id) { | Status DatasetOp::EoeReceived(int32_t worker_id) { | ||||
| std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| return (out_connector_->Add(static_cast<int>(worker_id), std::move(eoe_buffer))); | return (out_connector_->Add(static_cast<int>(worker_id), std::move(eoe_buffer))); | ||||
| } | } | ||||
| @@ -157,7 +157,7 @@ Status DatasetOp::EoeReceived(int32_t worker_id) { | |||||
| // The base class implementation simply flows the eof message to output. Derived classes | // The base class implementation simply flows the eof message to output. Derived classes | ||||
| // may override if they need to perform special eof handling. | // may override if they need to perform special eof handling. | ||||
| Status DatasetOp::EofReceived(int32_t worker_id) { | Status DatasetOp::EofReceived(int32_t worker_id) { | ||||
| std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer))); | return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer))); | ||||
| } | } | ||||
| @@ -225,7 +225,7 @@ Status DeviceQueueOp::SendDataToCPU() { | |||||
| MS_LOG(INFO) << "Device queue, sending data to CPU."; | MS_LOG(INFO) << "Device queue, sending data to CPU."; | ||||
| int64_t total_batch = 0; | int64_t total_batch = 0; | ||||
| std::unique_ptr<ChildIterator> child_iterator = mindspore::make_unique<ChildIterator>(this, 0, 0); | |||||
| std::unique_ptr<ChildIterator> child_iterator = std::make_unique<ChildIterator>(this, 0, 0); | |||||
| while (!(child_iterator->eof_handled())) { | while (!(child_iterator->eof_handled())) { | ||||
| TensorRow curr_row; | TensorRow curr_row; | ||||
| RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&curr_row)); | RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&curr_row)); | ||||
| @@ -179,7 +179,7 @@ Status MapOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get(), &keep_input_columns, &to_process_indices, &final_col_name_id_map, | RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get(), &keep_input_columns, &to_process_indices, &final_col_name_id_map, | ||||
| &input_columns, &output_columns)); | &input_columns, &output_columns)); | ||||
| std::unique_ptr<TensorQTable> new_tensor_table(mindspore::make_unique<TensorQTable>()); | |||||
| std::unique_ptr<TensorQTable> new_tensor_table(std::make_unique<TensorQTable>()); | |||||
| // Perform the compute function of TensorOp(s) and store the result in new_tensor_table. | // Perform the compute function of TensorOp(s) and store the result in new_tensor_table. | ||||
| RETURN_IF_NOT_OK(WorkerCompute(in_buffer.get(), to_process_indices, new_tensor_table.get(), keep_input_columns, | RETURN_IF_NOT_OK(WorkerCompute(in_buffer.get(), to_process_indices, new_tensor_table.get(), keep_input_columns, | ||||
| &input_columns, &output_columns)); | &input_columns, &output_columns)); | ||||
| @@ -48,7 +48,7 @@ Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) { | |||||
| // Instantiate the worker connector. This is the internal connector, not the operators | // Instantiate the worker connector. This is the internal connector, not the operators | ||||
| // output connector. It has single master consuming from it (num producers is 1), and the number | // output connector. It has single master consuming from it (num producers is 1), and the number | ||||
| // of workers is the defined count from the op. | // of workers is the defined count from the op. | ||||
| worker_connector_ = mindspore::make_unique<DbConnector>(num_workers_, num_producers_, worker_connector_size); | |||||
| worker_connector_ = std::make_unique<DbConnector>(num_workers_, num_producers_, worker_connector_size); | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -79,7 +79,7 @@ Status ProjectOp::Project(std::unique_ptr<DataBuffer> *data_buffer) { | |||||
| new_column_name_mapping[current_column] = i; | new_column_name_mapping[current_column] = i; | ||||
| projected_column_indices.push_back(column_name_mapping[current_column]); | projected_column_indices.push_back(column_name_mapping[current_column]); | ||||
| } | } | ||||
| std::unique_ptr<TensorQTable> new_tensor_table = mindspore::make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> new_tensor_table = std::make_unique<TensorQTable>(); | |||||
| while ((*data_buffer)->NumRows() > 0) { | while ((*data_buffer)->NumRows() > 0) { | ||||
| TensorRow current_row; | TensorRow current_row; | ||||
| RETURN_IF_NOT_OK((*data_buffer)->PopRow(¤t_row)); | RETURN_IF_NOT_OK((*data_buffer)->PopRow(¤t_row)); | ||||
| @@ -84,13 +84,13 @@ Status RenameOp::operator()() { | |||||
| // we got eoe, now try again until we get eof | // we got eoe, now try again until we get eof | ||||
| MS_LOG(INFO) << "Rename operator EOE Received."; | MS_LOG(INFO) << "Rename operator EOE Received."; | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| MS_LOG(DEBUG) << "Rename operator fetching buffer after EOE."; | MS_LOG(DEBUG) << "Rename operator fetching buffer after EOE."; | ||||
| RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); | RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); | ||||
| } // end of while eof loop | } // end of while eof loop | ||||
| MS_LOG(INFO) << "Rename opeerator EOF Received."; | MS_LOG(INFO) << "Rename opeerator EOF Received."; | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -70,7 +70,7 @@ ShuffleOp::ShuffleOp(int32_t shuffle_size, uint32_t shuffle_seed, int32_t op_con | |||||
| rng_(shuffle_seed), | rng_(shuffle_seed), | ||||
| buffer_counter_(0), | buffer_counter_(0), | ||||
| rows_per_buffer_(rows_per_buffer), | rows_per_buffer_(rows_per_buffer), | ||||
| shuffle_buffer_(mindspore::make_unique<TensorTable>()), | |||||
| shuffle_buffer_(std::make_unique<TensorTable>()), | |||||
| shuffle_last_row_idx_(0), | shuffle_last_row_idx_(0), | ||||
| shuffle_buffer_state_(kShuffleStateInit) {} | shuffle_buffer_state_(kShuffleStateInit) {} | ||||
| @@ -90,7 +90,7 @@ Status ShuffleOp::SelfReset() { | |||||
| shuffle_seed_ = distribution(random_device); | shuffle_seed_ = distribution(random_device); | ||||
| rng_ = std::mt19937_64(shuffle_seed_); | rng_ = std::mt19937_64(shuffle_seed_); | ||||
| } | } | ||||
| shuffle_buffer_ = mindspore::make_unique<TensorTable>(); | |||||
| shuffle_buffer_ = std::make_unique<TensorTable>(); | |||||
| buffer_counter_ = 0; | buffer_counter_ = 0; | ||||
| shuffle_last_row_idx_ = 0; | shuffle_last_row_idx_ = 0; | ||||
| shuffle_buffer_state_ = kShuffleStateInit; | shuffle_buffer_state_ = kShuffleStateInit; | ||||
| @@ -142,7 +142,7 @@ Status ShuffleOp::operator()() { | |||||
| // Create the child iterator to fetch our data from. | // Create the child iterator to fetch our data from. | ||||
| int32_t worker_id = 0; | int32_t worker_id = 0; | ||||
| int32_t child_idx = 0; | int32_t child_idx = 0; | ||||
| child_iterator_ = mindspore::make_unique<ChildIterator>(this, worker_id, child_idx); | |||||
| child_iterator_ = std::make_unique<ChildIterator>(this, worker_id, child_idx); | |||||
| // Main operator loop | // Main operator loop | ||||
| while (true) { | while (true) { | ||||
| @@ -161,7 +161,7 @@ Status ShuffleOp::operator()() { | |||||
| // Step 1) | // Step 1) | ||||
| // Create an output tensor table if one is not created yet. | // Create an output tensor table if one is not created yet. | ||||
| if (!new_buffer_table) { | if (!new_buffer_table) { | ||||
| new_buffer_table = mindspore::make_unique<TensorQTable>(); | |||||
| new_buffer_table = std::make_unique<TensorQTable>(); | |||||
| } | } | ||||
| // Step 2) | // Step 2) | ||||
| @@ -176,7 +176,7 @@ Status ShuffleOp::operator()() { | |||||
| // and send this buffer on it's way up the pipeline. Special case is if this is the | // and send this buffer on it's way up the pipeline. Special case is if this is the | ||||
| // last row then we also send it. | // last row then we also send it. | ||||
| if (new_buffer_table->size() == rows_per_buffer_ || shuffle_last_row_idx_ == 0) { | if (new_buffer_table->size() == rows_per_buffer_ || shuffle_last_row_idx_ == 0) { | ||||
| auto new_buffer = mindspore::make_unique<DataBuffer>(buffer_counter_, DataBuffer::kDeBFlagNone); | |||||
| auto new_buffer = std::make_unique<DataBuffer>(buffer_counter_, DataBuffer::kDeBFlagNone); | |||||
| new_buffer->set_tensor_table(std::move(new_buffer_table)); | new_buffer->set_tensor_table(std::move(new_buffer_table)); | ||||
| new_buffer->set_column_name_map(column_name_map_); | new_buffer->set_column_name_map(column_name_map_); | ||||
| buffer_counter_++; | buffer_counter_++; | ||||
| @@ -218,7 +218,7 @@ Status ShuffleOp::operator()() { | |||||
| // Since we overloaded eoeReceived function, we are responsible to flow the EOE up the | // Since we overloaded eoeReceived function, we are responsible to flow the EOE up the | ||||
| // pipepline manually now that we are done draining the shuffle buffer | // pipepline manually now that we are done draining the shuffle buffer | ||||
| MS_LOG(INFO) << "Shuffle operator sending EOE."; | MS_LOG(INFO) << "Shuffle operator sending EOE."; | ||||
| auto eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| auto eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | ||||
| // Do not wait for any reset to be flown down from operators above us. | // Do not wait for any reset to be flown down from operators above us. | ||||
| @@ -40,7 +40,7 @@ Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) { | |||||
| builder_sampler_ = std::make_shared<SequentialSampler>(); | builder_sampler_ = std::make_shared<SequentialSampler>(); | ||||
| } | } | ||||
| builder_schema_ = make_unique<DataSchema>(); | |||||
| builder_schema_ = std::make_unique<DataSchema>(); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | ||||
| // label is like this:0 1 0 0 1...... | // label is like this:0 1 0 0 1...... | ||||
| @@ -83,7 +83,7 @@ CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::stri | |||||
| col_name_map_[data_schema_->column(index).name()] = index; | col_name_map_[data_schema_->column(index).name()] = index; | ||||
| } | } | ||||
| attr_info_queue_ = make_unique<Queue<std::vector<std::string>>>(queue_size); | |||||
| attr_info_queue_ = std::make_unique<Queue<std::vector<std::string>>>(queue_size); | |||||
| io_block_queues_.Init(num_workers_, queue_size); | io_block_queues_.Init(num_workers_, queue_size); | ||||
| } | } | ||||
| @@ -311,7 +311,7 @@ Status CelebAOp::AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer) { | |||||
| row_count++; | row_count++; | ||||
| if (row_count % rows_per_buffer_ == 0) { | if (row_count % rows_per_buffer_ == 0) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[buff_count++ % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[buff_count++ % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| keys.clear(); | keys.clear(); | ||||
| } | } | ||||
| } | } | ||||
| @@ -320,21 +320,21 @@ Status CelebAOp::AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer) { | |||||
| if (!keys.empty()) { | if (!keys.empty()) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buff_count++) % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[(buff_count++) % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| for (int32_t i = 0; i < num_workers_; i++) { | for (int32_t i = 0; i < num_workers_; i++) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[i]->Add(std::move(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)))); | |||||
| io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset | } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | ||||
| wp_.Clear(); | wp_.Clear(); | ||||
| RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer)); | RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer)); | ||||
| @@ -349,17 +349,17 @@ Status CelebAOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | ||||
| while (io_block != nullptr) { | while (io_block != nullptr) { | ||||
| if (io_block->eoe() == true) { | if (io_block->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| buffer_id = worker_id; | buffer_id = worker_id; | ||||
| } else if (io_block->eof() == true) { | } else if (io_block->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| } else { | } else { | ||||
| std::vector<int64_t> keys; | std::vector<int64_t> keys; | ||||
| RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | ||||
| if (keys.empty()) { | if (keys.empty()) { | ||||
| return Status::OK(); // empty key is a quit signal for workers | return Status::OK(); // empty key is a quit signal for workers | ||||
| } | } | ||||
| std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | ||||
| buffer_id += num_workers_; | buffer_id += num_workers_; | ||||
| @@ -370,7 +370,7 @@ Status CelebAOp::WorkerEntry(int32_t worker_id) { | |||||
| } | } | ||||
| Status CelebAOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | Status CelebAOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | ||||
| std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>(); | |||||
| for (const auto &key : keys) { | for (const auto &key : keys) { | ||||
| TensorRow row; | TensorRow row; | ||||
| RETURN_IF_NOT_OK(LoadTensorRow(image_labels_vec_[key], &row)); | RETURN_IF_NOT_OK(LoadTensorRow(image_labels_vec_[key], &row)); | ||||
| @@ -47,7 +47,7 @@ Status CifarOp::Builder::Build(std::shared_ptr<CifarOp> *ptr) { | |||||
| if (sampler_ == nullptr) { | if (sampler_ == nullptr) { | ||||
| sampler_ = std::make_shared<SequentialSampler>(); | sampler_ = std::make_shared<SequentialSampler>(); | ||||
| } | } | ||||
| schema_ = make_unique<DataSchema>(); | |||||
| schema_ = std::make_unique<DataSchema>(); | |||||
| TensorShape scalar = TensorShape::CreateScalar(); | TensorShape scalar = TensorShape::CreateScalar(); | ||||
| RETURN_IF_NOT_OK(schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | RETURN_IF_NOT_OK(schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | ||||
| if (cifar_type_ == kCifar10) { | if (cifar_type_ == kCifar10) { | ||||
| @@ -91,7 +91,7 @@ CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const | |||||
| col_name_map_[data_schema_->column(i).name()] = i; | col_name_map_[data_schema_->column(i).name()] = i; | ||||
| } | } | ||||
| constexpr uint64_t kUtilQueueSize = 512; | constexpr uint64_t kUtilQueueSize = 512; | ||||
| cifar_raw_data_block_ = make_unique<Queue<std::vector<unsigned char>>>(kUtilQueueSize); | |||||
| cifar_raw_data_block_ = std::make_unique<Queue<std::vector<unsigned char>>>(kUtilQueueSize); | |||||
| io_block_queues_.Init(num_workers_, queue_size); | io_block_queues_.Init(num_workers_, queue_size); | ||||
| } | } | ||||
| @@ -114,7 +114,7 @@ Status CifarOp::operator()() { | |||||
| if (row_cnt_ >= num_samples_) break; // enough row read, break for loop | if (row_cnt_ >= num_samples_) break; // enough row read, break for loop | ||||
| if (row_cnt_ % rows_per_buffer_ == 0) { | if (row_cnt_ % rows_per_buffer_ == 0) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| keys.clear(); | keys.clear(); | ||||
| } | } | ||||
| } | } | ||||
| @@ -122,21 +122,21 @@ Status CifarOp::operator()() { | |||||
| } | } | ||||
| if (keys.empty() == false) { | if (keys.empty() == false) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| for (int32_t i = 0; i < num_workers_; i++) { | for (int32_t i = 0; i < num_workers_; i++) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset | } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | ||||
| wp_.Clear(); | wp_.Clear(); | ||||
| RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | ||||
| @@ -169,17 +169,17 @@ Status CifarOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | ||||
| while (io_block != nullptr) { | while (io_block != nullptr) { | ||||
| if (io_block->eoe() == true) { | if (io_block->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| buffer_id = worker_id; | buffer_id = worker_id; | ||||
| } else if (io_block->eof() == true) { | } else if (io_block->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| } else { | } else { | ||||
| std::vector<int64_t> keys; | std::vector<int64_t> keys; | ||||
| RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | ||||
| if (keys.empty() == true) { | if (keys.empty() == true) { | ||||
| return Status::OK(); // empty key is a quit signal for workers | return Status::OK(); // empty key is a quit signal for workers | ||||
| } | } | ||||
| std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | ||||
| buffer_id += num_workers_; | buffer_id += num_workers_; | ||||
| @@ -213,7 +213,7 @@ Status CifarOp::LoadTensorRow(uint64_t index, TensorRow *trow) { | |||||
| // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | ||||
| Status CifarOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | Status CifarOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | ||||
| std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>(); | |||||
| for (const int64_t &key : keys) { | for (const int64_t &key : keys) { | ||||
| TensorRow trow; | TensorRow trow; | ||||
| RETURN_IF_NOT_OK(LoadTensorRow(key, &trow)); | RETURN_IF_NOT_OK(LoadTensorRow(key, &trow)); | ||||
| @@ -173,9 +173,9 @@ Status GeneratorOp::operator()() { | |||||
| bool eof = false; | bool eof = false; | ||||
| while (!eof) { | while (!eof) { | ||||
| // Create new buffer each iteration | // Create new buffer each iteration | ||||
| fetched_buffer = mindspore::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone); | |||||
| fetched_buffer = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone); | |||||
| fetched_buffer->set_column_name_map(column_names_map_); | fetched_buffer->set_column_name_map(column_names_map_); | ||||
| std::unique_ptr<TensorQTable> fetched_table = mindspore::make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> fetched_table = std::make_unique<TensorQTable>(); | |||||
| bool eoe = false; | bool eoe = false; | ||||
| { | { | ||||
| py::gil_scoped_acquire gil_acquire; | py::gil_scoped_acquire gil_acquire; | ||||
| @@ -201,12 +201,12 @@ Status GeneratorOp::operator()() { | |||||
| if (eoe) { | if (eoe) { | ||||
| // Push out EOE upon StopIteration exception from generator | // Push out EOE upon StopIteration exception from generator | ||||
| MS_LOG(INFO) << "Generator operator sends out EOE."; | MS_LOG(INFO) << "Generator operator sends out EOE."; | ||||
| std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| // If last repeat or not repeated, push out EOF and exit master loop | // If last repeat or not repeated, push out EOF and exit master loop | ||||
| MS_LOG(INFO) << "Generator operator sends out EOF."; | MS_LOG(INFO) << "Generator operator sends out EOF."; | ||||
| std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); | ||||
| MS_LOG(INFO) << "Generator operator main execution loop complete."; | MS_LOG(INFO) << "Generator operator main execution loop complete."; | ||||
| eof = true; | eof = true; | ||||
| @@ -39,7 +39,7 @@ Status ImageFolderOp::Builder::Build(std::shared_ptr<ImageFolderOp> *ptr) { | |||||
| if (builder_sampler_ == nullptr) { | if (builder_sampler_ == nullptr) { | ||||
| builder_sampler_ = std::make_shared<SequentialSampler>(); | builder_sampler_ = std::make_shared<SequentialSampler>(); | ||||
| } | } | ||||
| builder_schema_ = make_unique<DataSchema>(); | |||||
| builder_schema_ = std::make_unique<DataSchema>(); | |||||
| TensorShape scalar = TensorShape::CreateScalar(); | TensorShape scalar = TensorShape::CreateScalar(); | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | ||||
| @@ -82,8 +82,8 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str | |||||
| for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { | for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { | ||||
| col_name_map_[data_schema_->column(i).name()] = i; | col_name_map_[data_schema_->column(i).name()] = i; | ||||
| } | } | ||||
| folder_name_queue_ = make_unique<Queue<std::string>>(num_wkrs * queue_size); | |||||
| image_name_queue_ = make_unique<Queue<FolderImagesPair>>(num_wkrs * queue_size); | |||||
| folder_name_queue_ = std::make_unique<Queue<std::string>>(num_wkrs * queue_size); | |||||
| image_name_queue_ = std::make_unique<Queue<FolderImagesPair>>(num_wkrs * queue_size); | |||||
| io_block_queues_.Init(num_workers_, queue_size); | io_block_queues_.Init(num_workers_, queue_size); | ||||
| } | } | ||||
| @@ -143,7 +143,7 @@ Status ImageFolderOp::operator()() { | |||||
| row_cnt_++; | row_cnt_++; | ||||
| if (row_cnt_ % rows_per_buffer_ == 0) { | if (row_cnt_ % rows_per_buffer_ == 0) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[buf_cnt_++ % num_workers_]->Add(make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[buf_cnt_++ % num_workers_]->Add(std::make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone))); | |||||
| keys.clear(); | keys.clear(); | ||||
| } | } | ||||
| } | } | ||||
| @@ -151,21 +151,21 @@ Status ImageFolderOp::operator()() { | |||||
| } | } | ||||
| if (keys.empty() == false) { | if (keys.empty() == false) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| std::unique_ptr<IOBlock> eoe_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe); | |||||
| std::unique_ptr<IOBlock> eof_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof); | |||||
| std::unique_ptr<IOBlock> eoe_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe); | |||||
| std::unique_ptr<IOBlock> eof_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof); | |||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); | ||||
| for (int32_t i = 0; i < num_workers_; ++i) { | for (int32_t i = 0; i < num_workers_; ++i) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { // not the last repeat. Sleep master thread, wait for the wake-up from reset | } else { // not the last repeat. Sleep master thread, wait for the wake-up from reset | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | ||||
| wp_.Clear(); | wp_.Clear(); | ||||
| RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | ||||
| @@ -182,15 +182,15 @@ Status ImageFolderOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | ||||
| while (io_block != nullptr) { | while (io_block != nullptr) { | ||||
| if (io_block->eoe() == true) { | if (io_block->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| buffer_id = worker_id; | buffer_id = worker_id; | ||||
| } else if (io_block->eof() == true) { | } else if (io_block->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| } else { | } else { | ||||
| std::vector<int64_t> keys; | std::vector<int64_t> keys; | ||||
| RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | ||||
| if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers | if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers | ||||
| std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | ||||
| buffer_id += num_workers_; | buffer_id += num_workers_; | ||||
| @@ -231,7 +231,7 @@ Status ImageFolderOp::LoadTensorRow(ImageLabelPair pairPtr, TensorRow *trow) { | |||||
| // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | ||||
| Status ImageFolderOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | Status ImageFolderOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | ||||
| std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>(); | |||||
| TensorRow trow; | TensorRow trow; | ||||
| for (const int64_t &key : keys) { | for (const int64_t &key : keys) { | ||||
| RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); | RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); | ||||
| @@ -40,7 +40,7 @@ Status ManifestOp::Builder::Build(std::shared_ptr<ManifestOp> *ptr) { | |||||
| if (builder_sampler_ == nullptr) { | if (builder_sampler_ == nullptr) { | ||||
| builder_sampler_ = std::make_shared<SequentialSampler>(); | builder_sampler_ = std::make_shared<SequentialSampler>(); | ||||
| } | } | ||||
| builder_schema_ = make_unique<DataSchema>(); | |||||
| builder_schema_ = std::make_unique<DataSchema>(); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| @@ -105,7 +105,7 @@ Status ManifestOp::AddIoBlock(std::unique_ptr<DataBuffer> *sampler_buffer) { | |||||
| row_cnt_++; | row_cnt_++; | ||||
| if (row_cnt_ % rows_per_buffer_ == 0) { | if (row_cnt_ % rows_per_buffer_ == 0) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| keys.clear(); | keys.clear(); | ||||
| } | } | ||||
| } | } | ||||
| @@ -113,21 +113,21 @@ Status ManifestOp::AddIoBlock(std::unique_ptr<DataBuffer> *sampler_buffer) { | |||||
| } | } | ||||
| if (keys.empty() == false) { | if (keys.empty() == false) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| for (int32_t i = 0; i < num_workers_; i++) { | for (int32_t i = 0; i < num_workers_; i++) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { | } else { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | ||||
| wp_.Clear(); | wp_.Clear(); | ||||
| RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer)); | RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer)); | ||||
| @@ -160,17 +160,17 @@ Status ManifestOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | ||||
| while (io_block != nullptr) { | while (io_block != nullptr) { | ||||
| if (io_block->eoe() == true) { | if (io_block->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| buffer_id = worker_id; | buffer_id = worker_id; | ||||
| } else if (io_block->eof() == true) { | } else if (io_block->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| } else { | } else { | ||||
| std::vector<int64_t> keys; | std::vector<int64_t> keys; | ||||
| RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | ||||
| if (keys.empty()) { | if (keys.empty()) { | ||||
| return Status::OK(); // empty key is a quit signal for workers | return Status::OK(); // empty key is a quit signal for workers | ||||
| } | } | ||||
| std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | ||||
| buffer_id += num_workers_; | buffer_id += num_workers_; | ||||
| @@ -227,7 +227,7 @@ Status ManifestOp::LoadTensorRow(const std::pair<std::string, std::vector<std::s | |||||
| // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | ||||
| Status ManifestOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | Status ManifestOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | ||||
| std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>(); | |||||
| for (const auto &key : keys) { | for (const auto &key : keys) { | ||||
| TensorRow trow; | TensorRow trow; | ||||
| RETURN_IF_NOT_OK(LoadTensorRow(image_labelname_[static_cast<size_t>(key)], &trow)); | RETURN_IF_NOT_OK(LoadTensorRow(image_labelname_[static_cast<size_t>(key)], &trow)); | ||||
| @@ -30,7 +30,6 @@ | |||||
| #include "dataset/engine/datasetops/dataset_op.h" | #include "dataset/engine/datasetops/dataset_op.h" | ||||
| #include "dataset/engine/db_connector.h" | #include "dataset/engine/db_connector.h" | ||||
| #include "dataset/engine/execution_tree.h" | #include "dataset/engine/execution_tree.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -96,18 +95,18 @@ MindRecordOp::MindRecordOp(int32_t num_mind_record_workers, int32_t rows_per_buf | |||||
| io_blk_queues_.Init(num_workers_, op_connector_queue_size); | io_blk_queues_.Init(num_workers_, op_connector_queue_size); | ||||
| if (!block_reader_) return; | if (!block_reader_) return; | ||||
| for (int32_t i = 0; i < num_workers_; ++i) { | for (int32_t i = 0; i < num_workers_; ++i) { | ||||
| block_buffer_.emplace_back(make_unique<std::vector<ShardTuple>>(std::vector<ShardTuple>{})); | |||||
| block_buffer_.emplace_back(std::make_unique<std::vector<ShardTuple>>(std::vector<ShardTuple>{})); | |||||
| } | } | ||||
| } | } | ||||
| // Private helper method to encapsulate some common construction/reset tasks | // Private helper method to encapsulate some common construction/reset tasks | ||||
| Status MindRecordOp::Init() { | Status MindRecordOp::Init() { | ||||
| shard_reader_ = mindspore::make_unique<ShardReader>(); | |||||
| shard_reader_ = std::make_unique<ShardReader>(); | |||||
| auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_); | auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_); | ||||
| CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed."); | CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed."); | ||||
| data_schema_ = mindspore::make_unique<DataSchema>(); | |||||
| data_schema_ = std::make_unique<DataSchema>(); | |||||
| std::vector<std::shared_ptr<Schema>> schema_vec = shard_reader_->get_shard_header()->get_schemas(); | std::vector<std::shared_ptr<Schema>> schema_vec = shard_reader_->get_shard_header()->get_schemas(); | ||||
| // check whether schema exists, if so use the first one | // check whether schema exists, if so use the first one | ||||
| @@ -144,7 +143,7 @@ Status MindRecordOp::Init() { | |||||
| } | } | ||||
| if (!load_all_cols) { | if (!load_all_cols) { | ||||
| std::unique_ptr<DataSchema> tmp_schema = make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> tmp_schema = std::make_unique<DataSchema>(); | |||||
| for (std::string colname : columns_to_load_) { | for (std::string colname : columns_to_load_) { | ||||
| CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), colname + ": doesn't exist"); | CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), colname + ": doesn't exist"); | ||||
| RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname]))); | RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname]))); | ||||
| @@ -298,7 +297,7 @@ Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr<T[]> *arr | |||||
| RETURN_IF_NOT_OK(GetFloat(&value, columns_json[column_name], use_double)); | RETURN_IF_NOT_OK(GetFloat(&value, columns_json[column_name], use_double)); | ||||
| *new_shape = TensorShape::CreateScalar(); | *new_shape = TensorShape::CreateScalar(); | ||||
| *array_data = mindspore::make_unique<T[]>(1); | |||||
| *array_data = std::make_unique<T[]>(1); | |||||
| (*array_data)[0] = value; | (*array_data)[0] = value; | ||||
| } else { | } else { | ||||
| if (column.hasShape()) { | if (column.hasShape()) { | ||||
| @@ -309,7 +308,7 @@ Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr<T[]> *arr | |||||
| } | } | ||||
| int idx = 0; | int idx = 0; | ||||
| *array_data = mindspore::make_unique<T[]>(new_shape->NumOfElements()); | |||||
| *array_data = std::make_unique<T[]>(new_shape->NumOfElements()); | |||||
| for (auto &element : columns_json[column_name]) { | for (auto &element : columns_json[column_name]) { | ||||
| T value = 0; | T value = 0; | ||||
| RETURN_IF_NOT_OK(GetFloat(&value, element, use_double)); | RETURN_IF_NOT_OK(GetFloat(&value, element, use_double)); | ||||
| @@ -350,7 +349,7 @@ Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr<T[]> *array | |||||
| RETURN_IF_NOT_OK(GetInt(&value, columns_json[column_name])); | RETURN_IF_NOT_OK(GetInt(&value, columns_json[column_name])); | ||||
| *new_shape = TensorShape::CreateScalar(); | *new_shape = TensorShape::CreateScalar(); | ||||
| *array_data = mindspore::make_unique<T[]>(1); | |||||
| *array_data = std::make_unique<T[]>(1); | |||||
| (*array_data)[0] = value; | (*array_data)[0] = value; | ||||
| } else { | } else { | ||||
| if (column.hasShape()) { | if (column.hasShape()) { | ||||
| @@ -361,7 +360,7 @@ Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr<T[]> *array | |||||
| } | } | ||||
| int idx = 0; | int idx = 0; | ||||
| *array_data = mindspore::make_unique<T[]>(new_shape->NumOfElements()); | |||||
| *array_data = std::make_unique<T[]>(new_shape->NumOfElements()); | |||||
| for (auto &element : columns_json[column_name]) { | for (auto &element : columns_json[column_name]) { | ||||
| T value = 0; | T value = 0; | ||||
| RETURN_IF_NOT_OK(GetInt(&value, element)); | RETURN_IF_NOT_OK(GetInt(&value, element)); | ||||
| @@ -431,12 +430,14 @@ Status MindRecordOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); | ||||
| while (io_block != nullptr) { | while (io_block != nullptr) { | ||||
| if (io_block->eoe() == true) { | if (io_block->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| RETURN_IF_NOT_OK( | |||||
| out_connector_->Add(worker_id, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); | ||||
| continue; | continue; | ||||
| } | } | ||||
| if (io_block->eof() == true) { | if (io_block->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| RETURN_IF_NOT_OK( | |||||
| out_connector_->Add(worker_id, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); | ||||
| continue; | continue; | ||||
| } | } | ||||
| @@ -486,9 +487,9 @@ Status MindRecordOp::WorkerEntry(int32_t worker_id) { | |||||
| Status MindRecordOp::GetBufferFromReader(std::unique_ptr<DataBuffer> *fetched_buffer, int64_t buffer_id, | Status MindRecordOp::GetBufferFromReader(std::unique_ptr<DataBuffer> *fetched_buffer, int64_t buffer_id, | ||||
| int32_t worker_id) { | int32_t worker_id) { | ||||
| *fetched_buffer = mindspore::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| *fetched_buffer = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| (*fetched_buffer)->set_column_name_map(column_name_mapping_); | (*fetched_buffer)->set_column_name_map(column_name_mapping_); | ||||
| std::unique_ptr<TensorQTable> tensor_table = mindspore::make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> tensor_table = std::make_unique<TensorQTable>(); | |||||
| for (int32_t i = 0; i < rows_per_buffer_; ++i) { | for (int32_t i = 0; i < rows_per_buffer_; ++i) { | ||||
| ShardTuple tupled_buffer; | ShardTuple tupled_buffer; | ||||
| if (block_reader_) { | if (block_reader_) { | ||||
| @@ -597,22 +598,22 @@ Status MindRecordOp::operator()() { | |||||
| for (int32_t i = 0; i < buffers_needed_; ++i) { | for (int32_t i = 0; i < buffers_needed_; ++i) { | ||||
| if (block_reader_) RETURN_IF_NOT_OK(FetchBlockBuffer(i)); | if (block_reader_) RETURN_IF_NOT_OK(FetchBlockBuffer(i)); | ||||
| std::vector<int64_t> keys(1, i); | std::vector<int64_t> keys(1, i); | ||||
| RETURN_IF_NOT_OK( | |||||
| io_blk_queues_[buf_cnt_++ % num_workers_]->Add(make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| RETURN_IF_NOT_OK(io_blk_queues_[buf_cnt_++ % num_workers_]->Add( | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| for (int32_t i = 0; i < num_workers_; i++) { | for (int32_t i = 0; i < num_workers_; i++) { | ||||
| RETURN_IF_NOT_OK( | |||||
| io_blk_queues_[i]->Add(std::move(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)))); | |||||
| RETURN_IF_NOT_OK(io_blk_queues_[i]->Add( | |||||
| std::move(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset | } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| // reset our buffer count and go to loop again. | // reset our buffer count and go to loop again. | ||||
| RETURN_IF_NOT_OK(shard_reader_wait_post_.Wait()); | RETURN_IF_NOT_OK(shard_reader_wait_post_.Wait()); | ||||
| @@ -656,7 +657,7 @@ Status MindRecordOp::LaunchThreadAndInitOp() { | |||||
| } | } | ||||
| Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *count) { | Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *count) { | ||||
| std::unique_ptr<ShardReader> shard_reader = mindspore::make_unique<ShardReader>(); | |||||
| std::unique_ptr<ShardReader> shard_reader = std::make_unique<ShardReader>(); | |||||
| MSRStatus rc = shard_reader->CountTotalRows(dataset_path, count); | MSRStatus rc = shard_reader->CountTotalRows(dataset_path, count); | ||||
| if (rc == MSRStatus::FAILED) { | if (rc == MSRStatus::FAILED) { | ||||
| RETURN_STATUS_UNEXPECTED("MindRecordOp count total rows failed."); | RETURN_STATUS_UNEXPECTED("MindRecordOp count total rows failed."); | ||||
| @@ -43,7 +43,7 @@ Status MnistOp::Builder::Build(std::shared_ptr<MnistOp> *ptr) { | |||||
| if (builder_sampler_ == nullptr) { | if (builder_sampler_ == nullptr) { | ||||
| builder_sampler_ = std::make_shared<SequentialSampler>(); | builder_sampler_ = std::make_shared<SequentialSampler>(); | ||||
| } | } | ||||
| builder_schema_ = make_unique<DataSchema>(); | |||||
| builder_schema_ = std::make_unique<DataSchema>(); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); | builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); | ||||
| TensorShape scalar = TensorShape::CreateScalar(); | TensorShape scalar = TensorShape::CreateScalar(); | ||||
| @@ -89,7 +89,7 @@ Status MnistOp::TraversalSampleIds(const std::shared_ptr<Tensor> &sample_ids, st | |||||
| row_cnt_++; | row_cnt_++; | ||||
| if (row_cnt_ % rows_per_buffer_ == 0) { | if (row_cnt_ % rows_per_buffer_ == 0) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone)))); | |||||
| keys->clear(); | keys->clear(); | ||||
| } | } | ||||
| } | } | ||||
| @@ -115,21 +115,21 @@ Status MnistOp::operator()() { | |||||
| } | } | ||||
| if (keys.empty() == false) { | if (keys.empty() == false) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof))); | |||||
| for (int32_t i = 0; i < num_workers_; ++i) { | for (int32_t i = 0; i < num_workers_; ++i) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { | } else { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks | ||||
| wp_.Clear(); | wp_.Clear(); | ||||
| RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | ||||
| @@ -145,15 +145,15 @@ Status MnistOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock)); | RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock)); | ||||
| while (iOBlock != nullptr) { | while (iOBlock != nullptr) { | ||||
| if (iOBlock->eoe() == true) { | if (iOBlock->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| buffer_id = worker_id; | buffer_id = worker_id; | ||||
| } else if (iOBlock->eof() == true) { | } else if (iOBlock->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))); | |||||
| } else { | } else { | ||||
| std::vector<int64_t> keys; | std::vector<int64_t> keys; | ||||
| RETURN_IF_NOT_OK(iOBlock->GetKeys(&keys)); | RETURN_IF_NOT_OK(iOBlock->GetKeys(&keys)); | ||||
| if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers | if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers | ||||
| std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | ||||
| buffer_id += num_workers_; | buffer_id += num_workers_; | ||||
| @@ -178,7 +178,7 @@ Status MnistOp::LoadTensorRow(const MnistLabelPair &mnist_pair, TensorRow *trow) | |||||
| // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | // Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer | ||||
| Status MnistOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | Status MnistOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | ||||
| std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>(); | |||||
| TensorRow trow; | TensorRow trow; | ||||
| for (const int64_t &key : keys) { | for (const int64_t &key : keys) { | ||||
| RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); | RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); | ||||
| @@ -309,8 +309,8 @@ Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *la | |||||
| CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num_images != num_labels"); | CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num_images != num_labels"); | ||||
| // The image size of the Mnist dataset is fixed at [28,28] | // The image size of the Mnist dataset is fixed at [28,28] | ||||
| int64_t size = kMnistImageRows * kMnistImageCols; | int64_t size = kMnistImageRows * kMnistImageCols; | ||||
| auto images_buf = mindspore::make_unique<char[]>(size * num_images); | |||||
| auto labels_buf = mindspore::make_unique<char[]>(num_images); | |||||
| auto images_buf = std::make_unique<char[]>(size * num_images); | |||||
| auto labels_buf = std::make_unique<char[]>(num_images); | |||||
| if (images_buf == nullptr || labels_buf == nullptr) { | if (images_buf == nullptr || labels_buf == nullptr) { | ||||
| std::string err_msg = "Fail to allocate memory for MNIST Buffer."; | std::string err_msg = "Fail to allocate memory for MNIST Buffer."; | ||||
| MS_LOG(ERROR) << err_msg.c_str(); | MS_LOG(ERROR) << err_msg.c_str(); | ||||
| @@ -52,9 +52,9 @@ Status DistributedSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer | |||||
| if (cnt_ > samples_per_buffer_) { | if (cnt_ > samples_per_buffer_) { | ||||
| RETURN_STATUS_UNEXPECTED("Distributed Sampler Error"); | RETURN_STATUS_UNEXPECTED("Distributed Sampler Error"); | ||||
| } else if (cnt_ == samples_per_buffer_) { | } else if (cnt_ == samples_per_buffer_) { | ||||
| (*out_buffer) = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| } else { | } else { | ||||
| (*out_buffer) = mindspore::make_unique<DataBuffer>(cnt_, DataBuffer::kDeBFlagNone); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(cnt_, DataBuffer::kDeBFlagNone); | |||||
| std::shared_ptr<Tensor> sample_ids; | std::shared_ptr<Tensor> sample_ids; | ||||
| RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_)); | RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_)); | ||||
| int64_t *id_ptr = reinterpret_cast<int64_t *>(sample_ids->StartAddr()); | int64_t *id_ptr = reinterpret_cast<int64_t *>(sample_ids->StartAddr()); | ||||
| @@ -63,7 +63,7 @@ Status DistributedSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer | |||||
| *(id_ptr++) = shuffle_ ? shuffle_vec_[static_cast<size_t>(next_id)] : next_id; | *(id_ptr++) = shuffle_ ? shuffle_vec_[static_cast<size_t>(next_id)] : next_id; | ||||
| } | } | ||||
| TensorRow row(1, sample_ids); | TensorRow row(1, sample_ids); | ||||
| (*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row)); | |||||
| (*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row)); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -53,9 +53,9 @@ Status PKSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { | |||||
| if (next_id_ > num_pk_samples_ || num_pk_samples_ == 0) { | if (next_id_ > num_pk_samples_ || num_pk_samples_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("Index out of bound in PKSampler"); | RETURN_STATUS_UNEXPECTED("Index out of bound in PKSampler"); | ||||
| } else if (next_id_ == num_pk_samples_) { | } else if (next_id_ == num_pk_samples_) { | ||||
| (*out_buffer) = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| } else { | } else { | ||||
| (*out_buffer) = mindspore::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone); | |||||
| std::shared_ptr<Tensor> sample_ids; | std::shared_ptr<Tensor> sample_ids; | ||||
| int64_t last_id = | int64_t last_id = | ||||
| (samples_per_buffer_ + next_id_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_ + next_id_; | (samples_per_buffer_ + next_id_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_ + next_id_; | ||||
| @@ -68,7 +68,7 @@ Status PKSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { | |||||
| *(id_ptr++) = samples[rnd_ind]; | *(id_ptr++) = samples[rnd_ind]; | ||||
| } | } | ||||
| TensorRow row(1, sample_ids); | TensorRow row(1, sample_ids); | ||||
| (*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row)); | |||||
| (*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row)); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -32,9 +32,9 @@ Status RandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { | |||||
| if (next_id_ > num_samples_) { | if (next_id_ > num_samples_) { | ||||
| RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error"); | RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error"); | ||||
| } else if (next_id_ == num_samples_) { | } else if (next_id_ == num_samples_) { | ||||
| (*out_buffer) = make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| } else { | } else { | ||||
| (*out_buffer) = make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone); | |||||
| std::shared_ptr<Tensor> sampleIds; | std::shared_ptr<Tensor> sampleIds; | ||||
| int64_t last_id = samples_per_buffer_ + next_id_ > num_samples_ ? num_samples_ : samples_per_buffer_ + next_id_; | int64_t last_id = samples_per_buffer_ + next_id_ > num_samples_ ? num_samples_ : samples_per_buffer_ + next_id_; | ||||
| RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_)); | RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_)); | ||||
| @@ -44,7 +44,7 @@ Status RandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { | |||||
| } | } | ||||
| next_id_ = last_id; | next_id_ = last_id; | ||||
| TensorRow row(1, sampleIds); | TensorRow row(1, sampleIds); | ||||
| (*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row)); | |||||
| (*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row)); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -61,7 +61,7 @@ Status RandomSampler::Init(const RandomAccessOp *op) { | |||||
| } | } | ||||
| std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_); | std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_); | ||||
| } else { | } else { | ||||
| dist = make_unique<std::uniform_int_distribution<int64_t>>(0, num_rows_ - 1); | |||||
| dist = std::make_unique<std::uniform_int_distribution<int64_t>>(0, num_rows_ - 1); | |||||
| } | } | ||||
| rnd_.seed(seed_++); | rnd_.seed(seed_++); | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -35,7 +35,7 @@ Status Sampler::CreateSamplerTensor(std::shared_ptr<Tensor> *sample_ids, int64_t | |||||
| } | } | ||||
| if (col_desc_ == nullptr) { | if (col_desc_ == nullptr) { | ||||
| // a ColDescriptor for Tensor that holds SampleIds | // a ColDescriptor for Tensor that holds SampleIds | ||||
| col_desc_ = make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1); | |||||
| col_desc_ = std::make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1); | |||||
| } | } | ||||
| TensorShape shape(std::vector<dsize_t>(1, num_elements)); | TensorShape shape(std::vector<dsize_t>(1, num_elements)); | ||||
| RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type())); | RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type())); | ||||
| @@ -27,7 +27,6 @@ | |||||
| #include "dataset/engine/data_buffer.h" | #include "dataset/engine/data_buffer.h" | ||||
| #include "dataset/engine/data_schema.h" | #include "dataset/engine/data_schema.h" | ||||
| #include "dataset/engine/datasetops/dataset_op.h" | #include "dataset/engine/datasetops/dataset_op.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -25,9 +25,9 @@ Status SequentialSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) | |||||
| if (next_id_ > num_samples_) { | if (next_id_ > num_samples_) { | ||||
| RETURN_STATUS_UNEXPECTED("Sequential Sampler Internal Error"); | RETURN_STATUS_UNEXPECTED("Sequential Sampler Internal Error"); | ||||
| } else if (next_id_ == num_samples_) { | } else if (next_id_ == num_samples_) { | ||||
| (*out_buffer) = make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| } else { | } else { | ||||
| (*out_buffer) = make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone); | |||||
| std::shared_ptr<Tensor> sampleIds; | std::shared_ptr<Tensor> sampleIds; | ||||
| int64_t lastId = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_; | int64_t lastId = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_; | ||||
| RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, lastId - next_id_)); | RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, lastId - next_id_)); | ||||
| @@ -36,7 +36,7 @@ Status SequentialSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) | |||||
| *(idPtr++) = next_id_++; | *(idPtr++) = next_id_++; | ||||
| } | } | ||||
| TensorRow row(1, sampleIds); | TensorRow row(1, sampleIds); | ||||
| (*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row)); | |||||
| (*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row)); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -64,9 +64,9 @@ Status SubsetRandomSampler::Reset() { | |||||
| Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { | Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) { | ||||
| // All samples have been drawn | // All samples have been drawn | ||||
| if (sample_id_ == indices_.size()) { | if (sample_id_ == indices_.size()) { | ||||
| (*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE); | |||||
| } else { | } else { | ||||
| (*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone); | |||||
| std::shared_ptr<Tensor> outputIds; | std::shared_ptr<Tensor> outputIds; | ||||
| int64_t last_id = sample_id_ + samples_per_buffer_; | int64_t last_id = sample_id_ + samples_per_buffer_; | ||||
| @@ -92,7 +92,7 @@ Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffe | |||||
| } | } | ||||
| // Create a TensorTable from that single tensor and push into DataBuffer | // Create a TensorTable from that single tensor and push into DataBuffer | ||||
| (*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, TensorRow(1, outputIds))); | |||||
| (*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, TensorRow(1, outputIds))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -46,10 +46,10 @@ Status WeightedRandomSampler::Init(const RandomAccessOp *op) { | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init WeightedRandomSampler"); | CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init WeightedRandomSampler"); | ||||
| if (!replacement_) { | if (!replacement_) { | ||||
| exp_dist_ = mindspore::make_unique<std::exponential_distribution<>>(1); | |||||
| exp_dist_ = std::make_unique<std::exponential_distribution<>>(1); | |||||
| InitOnePassSampling(); | InitOnePassSampling(); | ||||
| } else { | } else { | ||||
| discrete_dist_ = mindspore::make_unique<std::discrete_distribution<int64_t>>(weights_.begin(), weights_.end()); | |||||
| discrete_dist_ = std::make_unique<std::discrete_distribution<int64_t>>(weights_.begin(), weights_.end()); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -96,9 +96,9 @@ Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buf | |||||
| } | } | ||||
| if (sample_id_ == num_samples_) { | if (sample_id_ == num_samples_) { | ||||
| (*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE); | |||||
| } else { | } else { | ||||
| (*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone); | |||||
| (*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone); | |||||
| std::shared_ptr<Tensor> outputIds; | std::shared_ptr<Tensor> outputIds; | ||||
| int64_t last_id = sample_id_ + samples_per_buffer_; | int64_t last_id = sample_id_ + samples_per_buffer_; | ||||
| @@ -132,7 +132,7 @@ Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buf | |||||
| } | } | ||||
| // Create a TensorTable from that single tensor and push into DataBuffer | // Create a TensorTable from that single tensor and push into DataBuffer | ||||
| (*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, TensorRow(1, outputIds))); | |||||
| (*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, TensorRow(1, outputIds))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -24,7 +24,6 @@ | |||||
| #include "dataset/engine/datasetops/source/storage_client.h" | #include "dataset/engine/datasetops/source/storage_client.h" | ||||
| #include "dataset/engine/datasetops/source/storage_op.h" | #include "dataset/engine/datasetops/source/storage_op.h" | ||||
| #include "dataset/engine/datasetops/source/tf_client.h" | #include "dataset/engine/datasetops/source/tf_client.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/status.h" | #include "dataset/util/status.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -57,7 +56,7 @@ static Status CreateStorageClientSwitch( | |||||
| case DatasetType::kTf: { | case DatasetType::kTf: { | ||||
| // Construct the derived class TFClient, stored as base class StorageClient | // Construct the derived class TFClient, stored as base class StorageClient | ||||
| store_op->set_rows_per_buffer(32); | store_op->set_rows_per_buffer(32); | ||||
| *out_client = mindspore::make_unique<TFClient>(std::move(schema), store_op); | |||||
| *out_client = std::make_unique<TFClient>(std::move(schema), store_op); | |||||
| break; | break; | ||||
| } | } | ||||
| case DatasetType::kUnknown: | case DatasetType::kUnknown: | ||||
| @@ -83,7 +82,7 @@ Status StorageClient::CreateStorageClient( | |||||
| std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client | std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client | ||||
| // Make a new schema first. This only assigns the dataset type. It does not | // Make a new schema first. This only assigns the dataset type. It does not | ||||
| // create the columns yet. | // create the columns yet. | ||||
| auto new_schema = mindspore::make_unique<DataSchema>(); | |||||
| auto new_schema = std::make_unique<DataSchema>(); | |||||
| RETURN_IF_NOT_OK(new_schema->LoadDatasetType(dataset_schema_path)); | RETURN_IF_NOT_OK(new_schema->LoadDatasetType(dataset_schema_path)); | ||||
| RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); | RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -99,7 +98,7 @@ Status StorageClient::CreateStorageClient( | |||||
| std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client | std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client | ||||
| // The dataset type is passed in by the user. Create an empty schema with only | // The dataset type is passed in by the user. Create an empty schema with only | ||||
| // only the dataset type filled in and then create the client with it. | // only the dataset type filled in and then create the client with it. | ||||
| auto new_schema = mindspore::make_unique<DataSchema>(); | |||||
| auto new_schema = std::make_unique<DataSchema>(); | |||||
| new_schema->set_dataset_type(in_type); | new_schema->set_dataset_type(in_type); | ||||
| RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); | RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -147,7 +146,7 @@ Status StorageClient::AssignDatasetLayout(uint32_t num_rows, // In: Th | |||||
| // The current schema was just an empty one with only the dataset field populated. | // The current schema was just an empty one with only the dataset field populated. | ||||
| // Let's copy construct a new one that will be a copy of the input schema (releasing the old | // Let's copy construct a new one that will be a copy of the input schema (releasing the old | ||||
| // one) and then set the number of rows that the user requested. | // one) and then set the number of rows that the user requested. | ||||
| data_schema_ = mindspore::make_unique<DataSchema>(schema); | |||||
| data_schema_ = std::make_unique<DataSchema>(schema); | |||||
| CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647"); | CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647"); | ||||
| num_rows_in_dataset_ = num_rows; | num_rows_in_dataset_ = num_rows; | ||||
| @@ -303,7 +303,7 @@ Status StorageOp::init() { | |||||
| // For simplicity, we'll make both of them 3 so they are the same size. | // For simplicity, we'll make both of them 3 so they are the same size. | ||||
| int32_t action_queue_size = (buffers_needed / num_workers_) + 1; | int32_t action_queue_size = (buffers_needed / num_workers_) + 1; | ||||
| for (int32_t i = 0; i < num_workers_; ++i) { | for (int32_t i = 0; i < num_workers_; ++i) { | ||||
| auto new_queue = mindspore::make_unique<Queue<int32_t>>(action_queue_size); | |||||
| auto new_queue = std::make_unique<Queue<int32_t>>(action_queue_size); | |||||
| action_queue_.push_back(std::move(new_queue)); | action_queue_.push_back(std::move(new_queue)); | ||||
| } | } | ||||
| } | } | ||||
| @@ -483,10 +483,10 @@ Status StorageOp::operator()() { | |||||
| // Post the control message to tell the workers to stop waiting on action queue | // Post the control message to tell the workers to stop waiting on action queue | ||||
| // because we are done! | // because we are done! | ||||
| RETURN_IF_NOT_OK(this->PostEndOfData()); | RETURN_IF_NOT_OK(this->PostEndOfData()); | ||||
| std::unique_ptr<DataBuffer> eoeBuffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| std::unique_ptr<DataBuffer> eoeBuffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoeBuffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoeBuffer))); | ||||
| MS_LOG(INFO) << "StorageOp master: Flow end-of-data eof message."; | MS_LOG(INFO) << "StorageOp master: Flow end-of-data eof message."; | ||||
| std::unique_ptr<DataBuffer> eofBuffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| std::unique_ptr<DataBuffer> eofBuffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eofBuffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eofBuffer))); | ||||
| MS_LOG(INFO) << "StorageOp master: Main execution loop complete."; | MS_LOG(INFO) << "StorageOp master: Main execution loop complete."; | ||||
| done = true; // while loop exit | done = true; // while loop exit | ||||
| @@ -496,7 +496,7 @@ Status StorageOp::operator()() { | |||||
| // RepeatOp above us somewhere in the tree will re-init us with the data to fetch again | // RepeatOp above us somewhere in the tree will re-init us with the data to fetch again | ||||
| // once it gets the end-of-epoch message. | // once it gets the end-of-epoch message. | ||||
| MS_LOG(INFO) << "StorageOp master: Flow end-of-epoch eoe message."; | MS_LOG(INFO) << "StorageOp master: Flow end-of-epoch eoe message."; | ||||
| std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | ||||
| // reset our buffer count and go to loop again. | // reset our buffer count and go to loop again. | ||||
| @@ -27,7 +27,6 @@ | |||||
| #include "dataset/core/data_type.h" | #include "dataset/core/data_type.h" | ||||
| #include "dataset/engine/datasetops/source/storage_client.h" | #include "dataset/engine/datasetops/source/storage_client.h" | ||||
| #include "dataset/engine/data_schema.h" | #include "dataset/engine/data_schema.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -72,7 +71,7 @@ Status TFBuffer::Load() { | |||||
| } | } | ||||
| // Construct the Tensor table for this buffer. | // Construct the Tensor table for this buffer. | ||||
| tensor_table_ = mindspore::make_unique<TensorQTable>(); | |||||
| tensor_table_ = std::make_unique<TensorQTable>(); | |||||
| // At each position in the tensor table, instantiate the shared pointer to it's Tensor. | // At each position in the tensor table, instantiate the shared pointer to it's Tensor. | ||||
| uint32_t row = 0; | uint32_t row = 0; | ||||
| @@ -272,7 +271,7 @@ Status TFBuffer::LoadFloatList(const ColDescriptor ¤t_col, const dataengin | |||||
| // Identify how many values we have and then create a local array of these | // Identify how many values we have and then create a local array of these | ||||
| // to deserialize into | // to deserialize into | ||||
| *num_elements = float_list.value_size(); | *num_elements = float_list.value_size(); | ||||
| *float_array = mindspore::make_unique<float[]>(*num_elements); | |||||
| *float_array = std::make_unique<float[]>(*num_elements); | |||||
| for (int i = 0; i < float_list.value_size(); i++) { | for (int i = 0; i < float_list.value_size(); i++) { | ||||
| (*float_array)[i] = float_list.value(i); | (*float_array)[i] = float_list.value(i); | ||||
| } | } | ||||
| @@ -294,7 +293,7 @@ Status TFBuffer::LoadIntList(const ColDescriptor ¤t_col, const dataengine: | |||||
| // Identify how many values we have and then create a local array of these | // Identify how many values we have and then create a local array of these | ||||
| // to deserialize into | // to deserialize into | ||||
| *num_elements = int64_list.value_size(); | *num_elements = int64_list.value_size(); | ||||
| *int_array = mindspore::make_unique<int64_t[]>(*num_elements); | |||||
| *int_array = std::make_unique<int64_t[]>(*num_elements); | |||||
| for (int i = 0; i < int64_list.value_size(); i++) { | for (int i = 0; i < int64_list.value_size(); i++) { | ||||
| (*int_array)[i] = int64_list.value(i); | (*int_array)[i] = int64_list.value(i); | ||||
| } | } | ||||
| @@ -36,7 +36,6 @@ | |||||
| #include "dataset/engine/db_connector.h" | #include "dataset/engine/db_connector.h" | ||||
| #include "dataset/engine/execution_tree.h" | #include "dataset/engine/execution_tree.h" | ||||
| #include "dataset/engine/jagged_connector.h" | #include "dataset/engine/jagged_connector.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/path.h" | #include "dataset/util/path.h" | ||||
| #include "dataset/util/queue.h" | #include "dataset/util/queue.h" | ||||
| #include "dataset/util/random.h" | #include "dataset/util/random.h" | ||||
| @@ -54,7 +53,7 @@ TFReaderOp::Builder::Builder() | |||||
| builder_op_connector_size_ = config_manager->op_connector_size(); | builder_op_connector_size_ = config_manager->op_connector_size(); | ||||
| builder_rows_per_buffer_ = config_manager->rows_per_buffer(); | builder_rows_per_buffer_ = config_manager->rows_per_buffer(); | ||||
| builder_shuffle_files_ = false; | builder_shuffle_files_ = false; | ||||
| builder_data_schema_ = make_unique<DataSchema>(); | |||||
| builder_data_schema_ = std::make_unique<DataSchema>(); | |||||
| } | } | ||||
| Status TFReaderOp::Builder::ValidateInputs() const { | Status TFReaderOp::Builder::ValidateInputs() const { | ||||
| @@ -103,7 +102,7 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64 | |||||
| finished_reading_dataset_(false), | finished_reading_dataset_(false), | ||||
| shuffle_files_(shuffle_files), | shuffle_files_(shuffle_files), | ||||
| data_schema_(std::move(data_schema)), | data_schema_(std::move(data_schema)), | ||||
| filename_index_(make_unique<StringIndex>()), | |||||
| filename_index_(std::make_unique<StringIndex>()), | |||||
| load_io_block_queue_(true), | load_io_block_queue_(true), | ||||
| load_jagged_connector_(true), | load_jagged_connector_(true), | ||||
| num_rows_(0), | num_rows_(0), | ||||
| @@ -129,7 +128,7 @@ Status TFReaderOp::Init() { | |||||
| // parallel op base. | // parallel op base. | ||||
| RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); | RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); | ||||
| jagged_buffer_connector_ = mindspore::make_unique<JaggedConnector>(num_workers_, 1, worker_connector_size_); | |||||
| jagged_buffer_connector_ = std::make_unique<JaggedConnector>(num_workers_, 1, worker_connector_size_); | |||||
| // temporary: make size large enough to hold all files + EOE to avoid hangs | // temporary: make size large enough to hold all files + EOE to avoid hangs | ||||
| int32_t safe_queue_size = static_cast<int32_t>(std::ceil(dataset_files_list_.size() / num_workers_)) + 1; | int32_t safe_queue_size = static_cast<int32_t>(std::ceil(dataset_files_list_.size() / num_workers_)) + 1; | ||||
| @@ -229,7 +228,7 @@ Status TFReaderOp::operator()() { | |||||
| } | } | ||||
| // all workers finished reading for this epoch, and we have read all the data from all workers | // all workers finished reading for this epoch, and we have read all the data from all workers | ||||
| std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| @@ -241,7 +240,7 @@ Status TFReaderOp::operator()() { | |||||
| } | } | ||||
| } | } | ||||
| std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); | RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); | ||||
| RETURN_IF_NOT_OK(PostEndOfData()); | RETURN_IF_NOT_OK(PostEndOfData()); | ||||
| @@ -274,7 +273,7 @@ Status TFReaderOp::WorkerEntry(int32_t worker_id) { | |||||
| MS_LOG(INFO) << "TFReader operator worker " << worker_id << " loaded file " << filename << "."; | MS_LOG(INFO) << "TFReader operator worker " << worker_id << " loaded file " << filename << "."; | ||||
| } | } | ||||
| } else { | } else { | ||||
| std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(1, DataBuffer::kDeBFlagEOE); | |||||
| std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(1, DataBuffer::kDeBFlagEOE); | |||||
| RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(eoe_buffer))); | RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(eoe_buffer))); | ||||
| } | } | ||||
| @@ -288,7 +287,7 @@ Status TFReaderOp::WorkerEntry(int32_t worker_id) { | |||||
| // When the worker pops this control indicator, it will shut itself down gracefully. | // When the worker pops this control indicator, it will shut itself down gracefully. | ||||
| Status TFReaderOp::PostEndOfData() { | Status TFReaderOp::PostEndOfData() { | ||||
| for (int i = 0; i < num_workers_; ++i) { | for (int i = 0; i < num_workers_; ++i) { | ||||
| std::unique_ptr<FilenameBlock> eof = mindspore::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEof); | |||||
| std::unique_ptr<FilenameBlock> eof = std::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEof); | |||||
| RETURN_IF_NOT_OK(PushIoBlockQueue(i, std::move(eof))); | RETURN_IF_NOT_OK(PushIoBlockQueue(i, std::move(eof))); | ||||
| } | } | ||||
| @@ -299,7 +298,7 @@ Status TFReaderOp::PostEndOfData() { | |||||
| // pops this control indicator, it will wait until the next epoch starts and then resume execution. | // pops this control indicator, it will wait until the next epoch starts and then resume execution. | ||||
| Status TFReaderOp::PostEndOfEpoch(int32_t queue_index) { | Status TFReaderOp::PostEndOfEpoch(int32_t queue_index) { | ||||
| for (int i = 0; i < num_workers_; ++i) { | for (int i = 0; i < num_workers_; ++i) { | ||||
| std::unique_ptr<FilenameBlock> eoe = mindspore::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEoe); | |||||
| std::unique_ptr<FilenameBlock> eoe = std::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEoe); | |||||
| RETURN_IF_NOT_OK(PushIoBlockQueue((queue_index + i) % num_workers_, std::move(eoe))); | RETURN_IF_NOT_OK(PushIoBlockQueue((queue_index + i) % num_workers_, std::move(eoe))); | ||||
| } | } | ||||
| @@ -358,7 +357,7 @@ Status TFReaderOp::FillIOBlockShuffle(const std::vector<int64_t> &i_keys) { | |||||
| } | } | ||||
| if (!equal_rows_per_shard_) { | if (!equal_rows_per_shard_) { | ||||
| if (key_index++ % num_devices_ == device_id_) { | if (key_index++ % num_devices_ == device_id_) { | ||||
| auto ioBlock = make_unique<FilenameBlock>(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone); | |||||
| auto ioBlock = std::make_unique<FilenameBlock>(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone); | |||||
| RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | ||||
| queue_index = (queue_index + 1) % num_workers_; | queue_index = (queue_index + 1) % num_workers_; | ||||
| } | } | ||||
| @@ -367,7 +366,7 @@ Status TFReaderOp::FillIOBlockShuffle(const std::vector<int64_t> &i_keys) { | |||||
| auto file_it = filename_index_->Search(*it); | auto file_it = filename_index_->Search(*it); | ||||
| std::string file_name = file_it.value(); | std::string file_name = file_it.value(); | ||||
| if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { | if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { | ||||
| auto ioBlock = make_unique<FilenameBlock>(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone); | |||||
| auto ioBlock = std::make_unique<FilenameBlock>(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone); | |||||
| RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | ||||
| MS_LOG(DEBUG) << "File name " << *it << " start offset " << start_offset << " end_offset " << end_offset; | MS_LOG(DEBUG) << "File name " << *it << " start offset " << start_offset << " end_offset " << end_offset; | ||||
| queue_index = (queue_index + 1) % num_workers_; | queue_index = (queue_index + 1) % num_workers_; | ||||
| @@ -404,14 +403,15 @@ Status TFReaderOp::FillIOBlockNoShuffle() { | |||||
| } | } | ||||
| if (!equal_rows_per_shard_) { | if (!equal_rows_per_shard_) { | ||||
| if (key_index++ % num_devices_ == device_id_) { | if (key_index++ % num_devices_ == device_id_) { | ||||
| auto ioBlock = make_unique<FilenameBlock>(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone); | |||||
| auto ioBlock = | |||||
| std::make_unique<FilenameBlock>(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone); | |||||
| RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | ||||
| queue_index = (queue_index + 1) % num_workers_; | queue_index = (queue_index + 1) % num_workers_; | ||||
| } | } | ||||
| } else { | } else { | ||||
| std::string file_name = it.value(); | std::string file_name = it.value(); | ||||
| if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { | if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { | ||||
| auto ioBlock = make_unique<FilenameBlock>(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone); | |||||
| auto ioBlock = std::make_unique<FilenameBlock>(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone); | |||||
| RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); | ||||
| queue_index = (queue_index + 1) % num_workers_; | queue_index = (queue_index + 1) % num_workers_; | ||||
| } | } | ||||
| @@ -490,14 +490,13 @@ Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_off | |||||
| int64_t rows_read = 0; | int64_t rows_read = 0; | ||||
| int64_t rows_total = 0; | int64_t rows_total = 0; | ||||
| std::unique_ptr<DataBuffer> current_buffer = | |||||
| mindspore::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> current_buffer = std::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone); | |||||
| std::unordered_map<std::string, int32_t> column_name_map; | std::unordered_map<std::string, int32_t> column_name_map; | ||||
| for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { | for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { | ||||
| column_name_map[data_schema_->column(i).name()] = i; | column_name_map[data_schema_->column(i).name()] = i; | ||||
| } | } | ||||
| current_buffer->set_column_name_map(column_name_map); | current_buffer->set_column_name_map(column_name_map); | ||||
| std::unique_ptr<TensorQTable> new_tensor_table = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> new_tensor_table = std::make_unique<TensorQTable>(); | |||||
| while (reader.peek() != EOF) { | while (reader.peek() != EOF) { | ||||
| if (!load_jagged_connector_) { | if (!load_jagged_connector_) { | ||||
| @@ -532,9 +531,9 @@ Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_off | |||||
| current_buffer->set_tensor_table(std::move(new_tensor_table)); | current_buffer->set_tensor_table(std::move(new_tensor_table)); | ||||
| RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer))); | RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer))); | ||||
| current_buffer = make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone); | |||||
| current_buffer = std::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone); | |||||
| current_buffer->set_column_name_map(column_name_map); | current_buffer->set_column_name_map(column_name_map); | ||||
| new_tensor_table = make_unique<TensorQTable>(); | |||||
| new_tensor_table = std::make_unique<TensorQTable>(); | |||||
| rows_read = 0; | rows_read = 0; | ||||
| } | } | ||||
| } | } | ||||
| @@ -742,7 +741,7 @@ Status TFReaderOp::LoadFloatList(const ColDescriptor ¤t_col, const dataeng | |||||
| // Identify how many values we have and then create a local array of these | // Identify how many values we have and then create a local array of these | ||||
| // to deserialize into | // to deserialize into | ||||
| *num_elements = float_list.value_size(); | *num_elements = float_list.value_size(); | ||||
| *float_array = mindspore::make_unique<float[]>(*num_elements); | |||||
| *float_array = std::make_unique<float[]>(*num_elements); | |||||
| for (int i = 0; i < float_list.value_size(); ++i) { | for (int i = 0; i < float_list.value_size(); ++i) { | ||||
| (*float_array)[i] = float_list.value(i); | (*float_array)[i] = float_list.value(i); | ||||
| } | } | ||||
| @@ -38,7 +38,7 @@ Status VOCOp::Builder::Build(std::shared_ptr<VOCOp> *ptr) { | |||||
| if (builder_sampler_ == nullptr) { | if (builder_sampler_ == nullptr) { | ||||
| builder_sampler_ = std::make_shared<SequentialSampler>(); | builder_sampler_ = std::make_shared<SequentialSampler>(); | ||||
| } | } | ||||
| builder_schema_ = make_unique<DataSchema>(); | |||||
| builder_schema_ = std::make_unique<DataSchema>(); | |||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| @@ -85,7 +85,7 @@ Status VOCOp::TraverseSampleIds(const std::shared_ptr<Tensor> &sample_ids, std:: | |||||
| row_cnt_++; | row_cnt_++; | ||||
| if (row_cnt_ % rows_per_buffer_ == 0) { | if (row_cnt_ % rows_per_buffer_ == 0) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone)))); | |||||
| keys->clear(); | keys->clear(); | ||||
| } | } | ||||
| } | } | ||||
| @@ -110,21 +110,21 @@ Status VOCOp::operator()() { | |||||
| } | } | ||||
| if (keys.empty() == false) { | if (keys.empty() == false) { | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( | ||||
| make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone)))); | |||||
| } | } | ||||
| if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { | ||||
| std::unique_ptr<IOBlock> eoe_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe); | |||||
| std::unique_ptr<IOBlock> eof_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof); | |||||
| std::unique_ptr<IOBlock> eoe_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe); | |||||
| std::unique_ptr<IOBlock> eof_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof); | |||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); | ||||
| RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); | RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); | ||||
| for (int32_t i = 0; i < num_workers_; i++) { | for (int32_t i = 0; i < num_workers_; i++) { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } else { | } else { | ||||
| RETURN_IF_NOT_OK( | RETURN_IF_NOT_OK( | ||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe))); | |||||
| RETURN_IF_NOT_OK(wp_.Wait()); | RETURN_IF_NOT_OK(wp_.Wait()); | ||||
| wp_.Clear(); | wp_.Clear(); | ||||
| RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); | ||||
| @@ -164,7 +164,7 @@ Status VOCOp::LoadTensorRow(const std::string &image_id, TensorRow *trow) { | |||||
| } | } | ||||
| Status VOCOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | Status VOCOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) { | ||||
| std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>(); | |||||
| TensorRow trow; | TensorRow trow; | ||||
| for (const uint64_t &key : keys) { | for (const uint64_t &key : keys) { | ||||
| RETURN_IF_NOT_OK(this->LoadTensorRow(image_ids_[key], &trow)); | RETURN_IF_NOT_OK(this->LoadTensorRow(image_ids_[key], &trow)); | ||||
| @@ -182,15 +182,15 @@ Status VOCOp::WorkerEntry(int32_t worker_id) { | |||||
| RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); | ||||
| while (io_block != nullptr) { | while (io_block != nullptr) { | ||||
| if (io_block->eoe() == true) { | if (io_block->eoe() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))); | |||||
| buffer_id = worker_id; | buffer_id = worker_id; | ||||
| } else if (io_block->eof() == true) { | } else if (io_block->eof() == true) { | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| } else { | } else { | ||||
| std::vector<int64_t> keys; | std::vector<int64_t> keys; | ||||
| RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); | ||||
| if (keys.empty() == true) return Status::OK(); | if (keys.empty() == true) return Status::OK(); | ||||
| std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone); | |||||
| RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); | ||||
| buffer_id += num_workers_; | buffer_id += num_workers_; | ||||
| @@ -65,13 +65,13 @@ Status ZipOp::operator()() { | |||||
| // initialize the iterators | // initialize the iterators | ||||
| for (int32_t i = 0; i < children_num_; ++i) { | for (int32_t i = 0; i < children_num_; ++i) { | ||||
| // magic number 0 since Zip is not a parallel Op | // magic number 0 since Zip is not a parallel Op | ||||
| child_iterators_.push_back(mindspore::make_unique<ChildIterator>(this, 0, i)); | |||||
| child_iterators_.push_back(std::make_unique<ChildIterator>(this, 0, i)); | |||||
| } | } | ||||
| // Loop until eof is true | // Loop until eof is true | ||||
| while (!eof_) { | while (!eof_) { | ||||
| // Create tensor table and prepare it by fetching and packing the first zipped row into it. | // Create tensor table and prepare it by fetching and packing the first zipped row into it. | ||||
| std::unique_ptr<TensorQTable> curr_table = mindspore::make_unique<TensorQTable>(); | |||||
| std::unique_ptr<TensorQTable> curr_table = std::make_unique<TensorQTable>(); | |||||
| RETURN_IF_NOT_OK(prepare(curr_table.get())); | RETURN_IF_NOT_OK(prepare(curr_table.get())); | ||||
| // If an eof got picked up during the above prepare, then we're done | // If an eof got picked up during the above prepare, then we're done | ||||
| @@ -81,7 +81,7 @@ Status ZipOp::operator()() { | |||||
| while (!draining_) { | while (!draining_) { | ||||
| // 1. If a previous loop iteration sent the current table out, then create a new one. | // 1. If a previous loop iteration sent the current table out, then create a new one. | ||||
| if (curr_table == nullptr) { | if (curr_table == nullptr) { | ||||
| curr_table = mindspore::make_unique<TensorQTable>(); | |||||
| curr_table = std::make_unique<TensorQTable>(); | |||||
| } | } | ||||
| // 2 fill the table. Note: draining mode might get turned on if any of the child inputs were done | // 2 fill the table. Note: draining mode might get turned on if any of the child inputs were done | ||||
| @@ -89,8 +89,7 @@ Status ZipOp::operator()() { | |||||
| // 3 create and update buffer and send it to the out connector | // 3 create and update buffer and send it to the out connector | ||||
| if (!curr_table->empty()) { | if (!curr_table->empty()) { | ||||
| std::unique_ptr<DataBuffer> curr_buffer = | |||||
| mindspore::make_unique<DataBuffer>(buffer_id_, DataBuffer::kDeBFlagNone); | |||||
| std::unique_ptr<DataBuffer> curr_buffer = std::make_unique<DataBuffer>(buffer_id_, DataBuffer::kDeBFlagNone); | |||||
| curr_buffer->set_tensor_table(std::move(curr_table)); | curr_buffer->set_tensor_table(std::move(curr_table)); | ||||
| curr_buffer->set_column_name_map(col_name_id_map_); | curr_buffer->set_column_name_map(col_name_id_map_); | ||||
| MS_LOG(DEBUG) << "Zip operator finished one buffer, pushing, rows " << curr_buffer->NumRows() << ", cols " | MS_LOG(DEBUG) << "Zip operator finished one buffer, pushing, rows " << curr_buffer->NumRows() << ", cols " | ||||
| @@ -105,15 +104,14 @@ Status ZipOp::operator()() { | |||||
| MS_LOG(DEBUG) << "Zip operator is now draining child inputs."; | MS_LOG(DEBUG) << "Zip operator is now draining child inputs."; | ||||
| RETURN_IF_NOT_OK(drainPipeline()); | RETURN_IF_NOT_OK(drainPipeline()); | ||||
| // Now that we have drained child inputs, send the eoe up. | // Now that we have drained child inputs, send the eoe up. | ||||
| RETURN_IF_NOT_OK( | |||||
| out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)))); | |||||
| } | } | ||||
| } | } | ||||
| // 5 handle eof | // 5 handle eof | ||||
| // propagate eof here. | // propagate eof here. | ||||
| MS_LOG(INFO) << "Zip operator got EOF, propagating."; | MS_LOG(INFO) << "Zip operator got EOF, propagating."; | ||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)))); | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -65,7 +65,7 @@ class DbConnector : public Connector<std::unique_ptr<DataBuffer>> { | |||||
| RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); | RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); | ||||
| // Once an EOF message is encountered this flag will be set and we can return early. | // Once an EOF message is encountered this flag will be set and we can return early. | ||||
| if (end_of_file_) { | if (end_of_file_) { | ||||
| *result = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| *result = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF); | |||||
| } else { | } else { | ||||
| RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); | RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); | ||||
| if (*result == nullptr) { | if (*result == nullptr) { | ||||
| @@ -24,7 +24,7 @@ namespace mindspore { | |||||
| namespace dataset { | namespace dataset { | ||||
| // Constructor | // Constructor | ||||
| ExecutionTree::ExecutionTree() : id_count_(0) { | ExecutionTree::ExecutionTree() : id_count_(0) { | ||||
| tg_ = mindspore::make_unique<TaskGroup>(); | |||||
| tg_ = std::make_unique<TaskGroup>(); | |||||
| tree_state_ = kDeTStateInit; | tree_state_ = kDeTStateInit; | ||||
| prepare_flags_ = kDePrepNone; | prepare_flags_ = kDePrepNone; | ||||
| } | } | ||||
| @@ -24,7 +24,6 @@ | |||||
| #include "dataset/core/cv_tensor.h" | #include "dataset/core/cv_tensor.h" | ||||
| #include "dataset/core/tensor.h" | #include "dataset/core/tensor.h" | ||||
| #include "dataset/core/tensor_shape.h" | #include "dataset/core/tensor_shape.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/random.h" | #include "dataset/util/random.h" | ||||
| #define MAX_INT_PRECISION 16777216 // float int precision is 16777216 | #define MAX_INT_PRECISION 16777216 // float int precision is 16777216 | ||||
| @@ -376,7 +375,7 @@ Status HwcToChw(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output) | |||||
| int width = input_cv->shape()[1]; | int width = input_cv->shape()[1]; | ||||
| int num_channels = input_cv->shape()[2]; | int num_channels = input_cv->shape()[2]; | ||||
| auto output_cv = mindspore::make_unique<CVTensor>(TensorShape{num_channels, height, width}, input_cv->type()); | |||||
| auto output_cv = std::make_unique<CVTensor>(TensorShape{num_channels, height, width}, input_cv->type()); | |||||
| for (int i = 0; i < num_channels; ++i) { | for (int i = 0; i < num_channels; ++i) { | ||||
| cv::Mat mat; | cv::Mat mat; | ||||
| RETURN_IF_NOT_OK(output_cv->Mat({i}, &mat)); | RETURN_IF_NOT_OK(output_cv->Mat({i}, &mat)); | ||||
| @@ -20,7 +20,6 @@ | |||||
| #include "dataset/core/tensor.h" | #include "dataset/core/tensor.h" | ||||
| #include "dataset/kernels/tensor_op.h" | #include "dataset/kernels/tensor_op.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/status.h" | #include "dataset/util/status.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -16,7 +16,6 @@ | |||||
| #include "dataset/util/arena.h" | #include "dataset/util/arena.h" | ||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include <utility> | #include <utility> | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/system_pool.h" | #include "dataset/util/system_pool.h" | ||||
| #include "dataset/util/de_error.h" | #include "dataset/util/de_error.h" | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| @@ -18,10 +18,8 @@ | |||||
| #include <algorithm> | #include <algorithm> | ||||
| #include <limits> | #include <limits> | ||||
| #include <utility> | #include <utility> | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/de_error.h" | |||||
| #include "dataset/util/system_pool.h" | #include "dataset/util/system_pool.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -16,6 +16,13 @@ | |||||
| #ifndef DATASET_UTIL_DE_ERROR_H_ | #ifndef DATASET_UTIL_DE_ERROR_H_ | ||||
| #define DATASET_UTIL_DE_ERROR_H_ | #define DATASET_UTIL_DE_ERROR_H_ | ||||
| #ifdef DEBUG | |||||
| #include <cassert> | |||||
| #define DS_ASSERT(f) assert(f) | |||||
| #else | |||||
| #define DS_ASSERT(f) ((void)0) | |||||
| #endif | |||||
| #include <map> | #include <map> | ||||
| #include "utils/error_code.h" | #include "utils/error_code.h" | ||||
| @@ -18,8 +18,7 @@ | |||||
| #include <iostream> | #include <iostream> | ||||
| #include <iterator> | #include <iterator> | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/de_error.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -14,6 +14,7 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #include "dataset/util/lock.h" | #include "dataset/util/lock.h" | ||||
| #include "dataset/util/de_error.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -19,7 +19,6 @@ | |||||
| #include <atomic> | #include <atomic> | ||||
| #include <condition_variable> | #include <condition_variable> | ||||
| #include <mutex> | #include <mutex> | ||||
| #include "dataset/util/make_unique.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -1,37 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2019 Huawei Technologies Co., Ltd | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| */ | |||||
| #ifndef DATASET_UTIL_MAKE_UNIQUE_H_ | |||||
| #define DATASET_UTIL_MAKE_UNIQUE_H_ | |||||
| #ifdef DEBUG | |||||
| #include <cassert> | |||||
| #define DS_ASSERT(f) assert(f) | |||||
| #else | |||||
| #define DS_ASSERT(f) ((void)0) | |||||
| #endif | |||||
| #include <memory> | |||||
| #include <type_traits> | |||||
| #include <utility> | |||||
| #include "dataset/util/de_error.h" | |||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | |||||
| using std::make_unique; | |||||
| } // namespace mindspore | |||||
| #endif // DATASET_UTIL_MAKE_UNIQUE_H_ | |||||
| @@ -212,7 +212,7 @@ class QueueList { | |||||
| void Init(int num_queues, int capacity) { | void Init(int num_queues, int capacity) { | ||||
| queue_list_.reserve(num_queues); | queue_list_.reserve(num_queues); | ||||
| for (int i = 0; i < num_queues; i++) { | for (int i = 0; i < num_queues; i++) { | ||||
| queue_list_.emplace_back(mindspore::make_unique<Queue<T>>(capacity)); | |||||
| queue_list_.emplace_back(std::make_unique<Queue<T>>(capacity)); | |||||
| } | } | ||||
| } | } | ||||
| @@ -27,7 +27,6 @@ | |||||
| #include <string> | #include <string> | ||||
| #include <thread> | #include <thread> | ||||
| #include "dataset/util/de_error.h" | #include "dataset/util/de_error.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "dataset/util/intrp_resource.h" | #include "dataset/util/intrp_resource.h" | ||||
| #include "dataset/util/list.h" | #include "dataset/util/list.h" | ||||
| #include "dataset/util/memory_pool.h" | #include "dataset/util/memory_pool.h" | ||||
| @@ -17,7 +17,6 @@ | |||||
| #include "device/gpu/blocking_queue.h" | #include "device/gpu/blocking_queue.h" | ||||
| #include <chrono> | #include <chrono> | ||||
| #include "device/gpu/gpu_common.h" | #include "device/gpu/gpu_common.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "common/utils.h" | #include "common/utils.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -32,7 +31,7 @@ GpuQueue::GpuQueue(void *addr, size_t feature_size, size_t label_size, size_t ca | |||||
| stream_(0), | stream_(0), | ||||
| node_info_(nullptr) { | node_info_(nullptr) { | ||||
| CHECK_CUDA_RET_WITH_ERROR(cudaStreamCreate(&stream_), "Cuda Create Stream Failed"); | CHECK_CUDA_RET_WITH_ERROR(cudaStreamCreate(&stream_), "Cuda Create Stream Failed"); | ||||
| node_info_ = mindspore::make_unique<NodeInfo[]>(capacity); | |||||
| node_info_ = std::make_unique<NodeInfo[]>(capacity); | |||||
| } | } | ||||
| GpuQueue::~GpuQueue() { buffer_ = nullptr; } | GpuQueue::~GpuQueue() { buffer_ = nullptr; } | ||||
| @@ -23,7 +23,6 @@ | |||||
| #include <vector> | #include <vector> | ||||
| #include "kernel/gpu/gpu_kernel.h" | #include "kernel/gpu/gpu_kernel.h" | ||||
| #include "kernel/gpu/gpu_kernel_factory.h" | #include "kernel/gpu/gpu_kernel_factory.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "kernel/gpu/kernel_constants.h" | #include "kernel/gpu/kernel_constants.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -74,8 +73,8 @@ class BiasAddGpuKernel : public GpuKernel { | |||||
| // Expand to 4 dims for cudnnSetTensorNdDescriptorEx. | // Expand to 4 dims for cudnnSetTensorNdDescriptorEx. | ||||
| auto cudnn_dims = std::max(num_dims, 4UL); | auto cudnn_dims = std::max(num_dims, 4UL); | ||||
| std::unique_ptr<int[]> x_dims = mindspore::make_unique<int[]>(cudnn_dims); | |||||
| std::unique_ptr<int[]> b_dims = mindspore::make_unique<int[]>(cudnn_dims); | |||||
| std::unique_ptr<int[]> x_dims = std::make_unique<int[]>(cudnn_dims); | |||||
| std::unique_ptr<int[]> b_dims = std::make_unique<int[]>(cudnn_dims); | |||||
| for (size_t i = 0; i < cudnn_dims; i++) { | for (size_t i = 0; i < cudnn_dims; i++) { | ||||
| x_dims[i] = (i < num_dims) ? SizeToInt(x_shape[i]) : 1; | x_dims[i] = (i < num_dims) ? SizeToInt(x_shape[i]) : 1; | ||||
| b_dims[i] = (i == pos) ? SizeToInt(x_shape[i]) : 1; | b_dims[i] = (i == pos) ? SizeToInt(x_shape[i]) : 1; | ||||
| @@ -26,7 +26,6 @@ | |||||
| #include "kernel/gpu/gpu_kernel.h" | #include "kernel/gpu/gpu_kernel.h" | ||||
| #include "kernel/gpu/gpu_kernel_factory.h" | #include "kernel/gpu/gpu_kernel_factory.h" | ||||
| #include "kernel/gpu/kernel_constants.h" | #include "kernel/gpu/kernel_constants.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace kernel { | namespace kernel { | ||||
| @@ -84,8 +83,8 @@ class BiasAddGradGpuKernel : public GpuKernel { | |||||
| // Expand to 4 dims for cudnnSetTensorNdDescriptorEx. | // Expand to 4 dims for cudnnSetTensorNdDescriptorEx. | ||||
| auto cudnn_dims = std::max(num_dims, 4UL); | auto cudnn_dims = std::max(num_dims, 4UL); | ||||
| std::unique_ptr<int[]> dy_dims = mindspore::make_unique<int[]>(cudnn_dims); | |||||
| std::unique_ptr<int[]> db_dims = mindspore::make_unique<int[]>(cudnn_dims); | |||||
| std::unique_ptr<int[]> dy_dims = std::make_unique<int[]>(cudnn_dims); | |||||
| std::unique_ptr<int[]> db_dims = std::make_unique<int[]>(cudnn_dims); | |||||
| for (size_t i = 0; i < cudnn_dims; i++) { | for (size_t i = 0; i < cudnn_dims; i++) { | ||||
| dy_dims[i] = (i < num_dims) ? SizeToInt(dy_shape[i]) : 1; | dy_dims[i] = (i < num_dims) ? SizeToInt(dy_shape[i]) : 1; | ||||
| db_dims[i] = (i == pos) ? SizeToInt(dy_shape[i]) : 1; | db_dims[i] = (i == pos) ? SizeToInt(dy_shape[i]) : 1; | ||||
| @@ -22,7 +22,6 @@ | |||||
| #include <memory> | #include <memory> | ||||
| #include "kernel/gpu/gpu_kernel.h" | #include "kernel/gpu/gpu_kernel.h" | ||||
| #include "kernel/gpu/gpu_kernel_factory.h" | #include "kernel/gpu/gpu_kernel_factory.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "kernel/gpu/kernel_constants.h" | #include "kernel/gpu/kernel_constants.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -144,8 +143,8 @@ class LstmGpuKernel : public GpuKernel { | |||||
| int x_dims[3]{batch_size_, input_size_, 1}; | int x_dims[3]{batch_size_, input_size_, 1}; | ||||
| int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; | int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; | ||||
| x_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| x_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| for (size_t i = 0; i < IntToSize(seq_len_); ++i) { | for (size_t i = 0; i < IntToSize(seq_len_); ++i) { | ||||
| CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); | CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); | ||||
| @@ -23,7 +23,6 @@ | |||||
| #include "kernel/gpu/gpu_kernel.h" | #include "kernel/gpu/gpu_kernel.h" | ||||
| #include "kernel/gpu/gpu_kernel_factory.h" | #include "kernel/gpu/gpu_kernel_factory.h" | ||||
| #include "kernel/gpu/kernel_constants.h" | #include "kernel/gpu/kernel_constants.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace kernel { | namespace kernel { | ||||
| @@ -212,9 +211,9 @@ class LstmGradDataGpuKernel : public GpuKernel { | |||||
| int x_dims[3]{batch_size_, input_size_, 1}; | int x_dims[3]{batch_size_, input_size_, 1}; | ||||
| int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; | int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; | ||||
| dx_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| dy_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| dx_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| dy_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| for (size_t i = 0; i < IntToSize(seq_len_); ++i) { | for (size_t i = 0; i < IntToSize(seq_len_); ++i) { | ||||
| CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_[i]), "create x_desc failed"); | CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_[i]), "create x_desc failed"); | ||||
| @@ -22,7 +22,6 @@ | |||||
| #include <memory> | #include <memory> | ||||
| #include "kernel/gpu/gpu_kernel.h" | #include "kernel/gpu/gpu_kernel.h" | ||||
| #include "kernel/gpu/gpu_kernel_factory.h" | #include "kernel/gpu/gpu_kernel_factory.h" | ||||
| #include "dataset/util/make_unique.h" | |||||
| #include "kernel/gpu/kernel_constants.h" | #include "kernel/gpu/kernel_constants.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace kernel { | namespace kernel { | ||||
| @@ -169,8 +168,8 @@ class LstmGradWeightGpuKernel : public GpuKernel { | |||||
| int x_dims[3]{batch_size_, input_size_, 1}; | int x_dims[3]{batch_size_, input_size_, 1}; | ||||
| int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; | int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; | ||||
| x_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| x_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_); | |||||
| for (size_t i = 0; i < IntToSize(seq_len_); ++i) { | for (size_t i = 0; i < IntToSize(seq_len_); ++i) { | ||||
| CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); | CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); | ||||
| @@ -116,7 +116,7 @@ TEST_F(MindDataTestCelebaDataset, TestCelebaRepeat) { | |||||
| TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) { | TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) { | ||||
| std::vector<int64_t> indices({1}); | std::vector<int64_t> indices({1}); | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices); | |||||
| uint32_t expect_labels[1][40] = {{0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}}; | uint32_t expect_labels[1][40] = {{0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}}; | ||||
| std::string dir = datasets_root_path_ + "/testCelebAData/"; | std::string dir = datasets_root_path_ + "/testCelebAData/"; | ||||
| uint32_t count = 0; | uint32_t count = 0; | ||||
| @@ -92,7 +92,7 @@ TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar10) { | |||||
| TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) { | TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) { | ||||
| uint32_t original_seed = GlobalContext::config_manager()->seed(); | uint32_t original_seed = GlobalContext::config_manager()->seed(); | ||||
| GlobalContext::config_manager()->set_seed(0); | GlobalContext::config_manager()->set_seed(0); | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<RandomSampler>(true, 12); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<RandomSampler>(true, 12); | |||||
| std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; | ||||
| auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler), 100)}); | auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler), 100)}); | ||||
| tree->Prepare(); | tree->Prepare(); | ||||
| @@ -138,7 +138,7 @@ TEST_F(MindDataTestImageFolderSampler, TestRandomImageFolder) { | |||||
| TEST_F(MindDataTestImageFolderSampler, TestRandomSamplerImageFolder) { | TEST_F(MindDataTestImageFolderSampler, TestRandomSamplerImageFolder) { | ||||
| int32_t original_seed = GlobalContext::config_manager()->seed(); | int32_t original_seed = GlobalContext::config_manager()->seed(); | ||||
| GlobalContext::config_manager()->set_seed(0); | GlobalContext::config_manager()->set_seed(0); | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<RandomSampler>(true, 12); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<RandomSampler>(true, 12); | |||||
| int32_t res[] = {2, 2, 2, 3, 2, 3, 2, 3, 1, 2, 2, 1}; // ground truth label | int32_t res[] = {2, 2, 2, 3, 2, 3, 2, 3, 1, 2, 2, 1}; // ground truth label | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); | auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); | ||||
| @@ -200,7 +200,7 @@ TEST_F(MindDataTestImageFolderSampler, TestSequentialImageFolderWithRepeatBatch) | |||||
| TEST_F(MindDataTestImageFolderSampler, TestSubsetRandomSamplerImageFolder) { | TEST_F(MindDataTestImageFolderSampler, TestSubsetRandomSamplerImageFolder) { | ||||
| // id range 0 - 10 is label 0, and id range 11 - 21 is label 1 | // id range 0 - 10 is label 0, and id range 11 - 21 is label 1 | ||||
| std::vector<int64_t> indices({0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11}); | std::vector<int64_t> indices({0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11}); | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices); | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| // Expect 6 samples for label 0 and 1 | // Expect 6 samples for label 0 and 1 | ||||
| int res[2] = {6, 6}; | int res[2] = {6, 6}; | ||||
| @@ -238,7 +238,7 @@ TEST_F(MindDataTestImageFolderSampler, TestWeightedRandomSamplerImageFolder) { | |||||
| // create sampler with replacement = replacement | // create sampler with replacement = replacement | ||||
| std::unique_ptr<Sampler> sampler = | std::unique_ptr<Sampler> sampler = | ||||
| mindspore::make_unique<WeightedRandomSampler>(weights, num_samples, true, samples_per_buffer); | |||||
| std::make_unique<WeightedRandomSampler>(weights, num_samples, true, samples_per_buffer); | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); | auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); | ||||
| @@ -295,7 +295,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderClassIndex) { | |||||
| } | } | ||||
| TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) { | TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) { | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(11, 10, false); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(11, 10, false); | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler)), Repeat(4)}); | auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler)), Repeat(4)}); | ||||
| tree->Prepare(); | tree->Prepare(); | ||||
| @@ -322,7 +322,7 @@ TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) { | |||||
| } | } | ||||
| TEST_F(MindDataTestImageFolderSampler, TestPKSamplerImageFolder) { | TEST_F(MindDataTestImageFolderSampler, TestPKSamplerImageFolder) { | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<PKSampler>(3, false, 4); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<PKSampler>(3, false, 4); | |||||
| int32_t res[] = {0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; // ground truth label | int32_t res[] = {0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; // ground truth label | ||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); | auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); | ||||
| @@ -431,7 +431,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderDatasetSize) { | |||||
| } | } | ||||
| TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) { | TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) { | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(4, 0, false); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(4, 0, false); | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode | // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode | ||||
| auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler), {}, 5)}); | auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler), {}, 5)}); | ||||
| @@ -460,7 +460,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) { | |||||
| } | } | ||||
| TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding2) { | TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding2) { | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(4, 3, false); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(4, 3, false); | |||||
| std::string folder_path = datasets_root_path_ + "/testPK/data"; | std::string folder_path = datasets_root_path_ + "/testPK/data"; | ||||
| // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode | // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode | ||||
| auto tree = Build({ImageFolder(16, 16, 32, folder_path, false, std::move(sampler), {}, 12)}); | auto tree = Build({ImageFolder(16, 16, 32, folder_path, false, std::move(sampler), {}, 12)}); | ||||
| @@ -86,7 +86,7 @@ TEST_F(MindDataTestManifest, TestSequentialManifestWithRepeat) { | |||||
| TEST_F(MindDataTestManifest, TestSubsetRandomSamplerManifest) { | TEST_F(MindDataTestManifest, TestSubsetRandomSamplerManifest) { | ||||
| std::vector<int64_t> indices({1}); | std::vector<int64_t> indices({1}); | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices); | |||||
| std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; | std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; | ||||
| // Expect 6 samples for label 0 and 1 | // Expect 6 samples for label 0 and 1 | ||||
| auto tree = Build({Manifest(16, 2, 32, file, "train", std::move(sampler))}); | auto tree = Build({Manifest(16, 2, 32, file, "train", std::move(sampler))}); | ||||
| @@ -45,7 +45,7 @@ TEST_F(MindDataTestProjectOp, TestProjectProject) { | |||||
| .SetRowsPerBuffer(16) | .SetRowsPerBuffer(16) | ||||
| .SetWorkerConnectorSize(16) | .SetWorkerConnectorSize(16) | ||||
| .SetNumWorkers(16); | .SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk()); | Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk()); | ||||
| @@ -74,7 +74,7 @@ TEST_F(MindDataTestStandAloneSampler, TestDistributedSampler) { | |||||
| std::unique_ptr<DataBuffer> db; | std::unique_ptr<DataBuffer> db; | ||||
| std::shared_ptr<Tensor> tensor; | std::shared_ptr<Tensor> tensor; | ||||
| for (int i = 0; i < 6; i++) { | for (int i = 0; i < 6; i++) { | ||||
| std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(3, i % 3, (i < 3 ? false : true)); | |||||
| std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(3, i % 3, (i < 3 ? false : true)); | |||||
| sampler->Init(&mock); | sampler->Init(&mock); | ||||
| sampler->GetNextBuffer(&db); | sampler->GetNextBuffer(&db); | ||||
| db->GetTensor(&tensor, 0, 0); | db->GetTensor(&tensor, 0, 0); | ||||
| @@ -48,7 +48,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasic1) { | |||||
| builder.SetDatasetFilesList({dataset_path}) | builder.SetDatasetFilesList({dataset_path}) | ||||
| .SetRowsPerBuffer(16) | .SetRowsPerBuffer(16) | ||||
| .SetNumWorkers(16); | .SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc = builder.Build(&my_tfreader_op); | Status rc = builder.Build(&my_tfreader_op); | ||||
| @@ -102,7 +102,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeRowsPerBuffer) { | |||||
| builder.SetDatasetFilesList({dataset_path}) | builder.SetDatasetFilesList({dataset_path}) | ||||
| .SetRowsPerBuffer(500) | .SetRowsPerBuffer(500) | ||||
| .SetNumWorkers(16); | .SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc = builder.Build(&my_tfreader_op); | Status rc = builder.Build(&my_tfreader_op); | ||||
| @@ -156,7 +156,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSmallRowsPerBuffer) { | |||||
| builder.SetDatasetFilesList({dataset_path}) | builder.SetDatasetFilesList({dataset_path}) | ||||
| .SetRowsPerBuffer(1) | .SetRowsPerBuffer(1) | ||||
| .SetNumWorkers(16); | .SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc = builder.Build(&my_tfreader_op); | Status rc = builder.Build(&my_tfreader_op); | ||||
| @@ -211,7 +211,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeQueueSize) { | |||||
| .SetWorkerConnectorSize(1) | .SetWorkerConnectorSize(1) | ||||
| .SetRowsPerBuffer(16) | .SetRowsPerBuffer(16) | ||||
| .SetNumWorkers(16); | .SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc = builder.Build(&my_tfreader_op); | Status rc = builder.Build(&my_tfreader_op); | ||||
| @@ -265,7 +265,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderOneThread) { | |||||
| builder.SetDatasetFilesList({dataset_path}) | builder.SetDatasetFilesList({dataset_path}) | ||||
| .SetRowsPerBuffer(16) | .SetRowsPerBuffer(16) | ||||
| .SetNumWorkers(1); | .SetNumWorkers(1); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc = builder.Build(&my_tfreader_op); | Status rc = builder.Build(&my_tfreader_op); | ||||
| @@ -321,7 +321,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderRepeat) { | |||||
| .SetRowsPerBuffer(16) | .SetRowsPerBuffer(16) | ||||
| .SetWorkerConnectorSize(16) | .SetWorkerConnectorSize(16) | ||||
| .SetNumWorkers(16); | .SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| Status rc= builder.Build(&my_tfreader_op); | Status rc= builder.Build(&my_tfreader_op); | ||||
| @@ -379,7 +379,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSchemaConstructor) { | |||||
| std::string dataset_path; | std::string dataset_path; | ||||
| dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; | dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; | ||||
| std::unique_ptr<DataSchema> data_schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> data_schema = std::make_unique<DataSchema>(); | |||||
| std::vector<std::string> columns_to_load; | std::vector<std::string> columns_to_load; | ||||
| columns_to_load.push_back("col_sint32"); | columns_to_load.push_back("col_sint32"); | ||||
| columns_to_load.push_back("col_binary"); | columns_to_load.push_back("col_binary"); | ||||
| @@ -445,7 +445,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Row) { | |||||
| std::shared_ptr<TFReaderOp> my_tfreader_op; | std::shared_ptr<TFReaderOp> my_tfreader_op; | ||||
| TFReaderOp::Builder builder; | TFReaderOp::Builder builder; | ||||
| builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); | builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| @@ -503,7 +503,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Buffer) { | |||||
| std::shared_ptr<TFReaderOp> my_tfreader_op; | std::shared_ptr<TFReaderOp> my_tfreader_op; | ||||
| TFReaderOp::Builder builder; | TFReaderOp::Builder builder; | ||||
| builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); | builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||
| @@ -561,7 +561,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake7Rows) { | |||||
| std::shared_ptr<TFReaderOp> my_tfreader_op; | std::shared_ptr<TFReaderOp> my_tfreader_op; | ||||
| TFReaderOp::Builder builder; | TFReaderOp::Builder builder; | ||||
| builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); | builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); | ||||
| std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>(); | |||||
| std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); | |||||
| schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {}); | schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {}); | ||||
| builder.SetDataSchema(std::move(schema)); | builder.SetDataSchema(std::move(schema)); | ||||