diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index fb6a03bdda..a6f6d6fcf5 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -20,12 +20,13 @@ #include #include +#include "minddata/dataset/core/tensor.h" #include "minddata/dataset/engine/runtime_context.h" #include "minddata/dataset/include/samplers.h" #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/status.h" - +#include "minddata/dataset/include/type_id.h" #include "minddata/dataset/core/client.h" #include "minddata/dataset/engine/consumers/tree_consumer.h" @@ -101,6 +102,37 @@ namespace mindspore { namespace dataset { +// convert MSTensorVec to DE TensorRow, return empty if fails +TensorRow VecToRow(const MSTensorVec &v) { + TensorRow row; + row.reserve(v.size()); + for (const MSTensor &t : v) { + std::shared_ptr rt; + Status rc = Tensor::CreateFromMSTensor(t, &rt); + if (rc.IsError()) { + MS_LOG_ERROR << "Convert from MSTensor to DETensor failed:" << rc.ToString() << "."; + return {}; + } + row.emplace_back(rt); + } + return row; +} + +// convert DE TensorRow to MSTensorVec, won't fail +MSTensorVec RowToVec(const TensorRow &v) { + MSTensorVec rv; + rv.reserve(v.size()); + std::transform(v.begin(), v.end(), std::back_inserter(rv), [](std::shared_ptr t) -> MSTensor { + return mindspore::MSTensor(std::make_shared(t)); + }); + return rv; +} + +// Convert a std::function to std::function with this helper +TensorRow FuncPtrConverter(std::function func, TensorRow in_row) { + return VecToRow(func(RowToVec(in_row))); +} + // Function to create the iterator, which will build and launch the execution tree. std::shared_ptr Dataset::CreateIteratorCharIF(std::vector> columns, int32_t num_epochs) { std::shared_ptr iter; @@ -228,22 +260,29 @@ int64_t Dataset::GetDatasetSize(bool estimate) { return dataset_size; } -std::vector Dataset::GetOutputTypes() { +std::vector Dataset::GetOutputTypes() { std::vector types; std::unique_ptr runtime_context = std::make_unique(); RETURN_SECOND_IF_ERROR(runtime_context->Init(), {}); RETURN_SECOND_IF_ERROR(tree_getters_->Init(this->IRNode()), {}); RETURN_SECOND_IF_ERROR(tree_getters_->GetOutputTypes(&types), {}); - return types; + std::vector ret_types; + std::transform( + types.begin(), types.end(), std::back_inserter(ret_types), + [](const DataType &d) -> mindspore::DataType { return static_cast(DETypeToMSType(d)); }); + return ret_types; } -std::vector Dataset::GetOutputShapes() { +std::vector> Dataset::GetOutputShapes() { std::vector shapes; std::unique_ptr runtime_context = std::make_unique(); RETURN_SECOND_IF_ERROR(runtime_context->Init(), {}); RETURN_SECOND_IF_ERROR(tree_getters_->Init(this->IRNode()), {}); RETURN_SECOND_IF_ERROR(tree_getters_->GetOutputShapes(&shapes), {}); - return shapes; + std::vector> ret_shapes; + std::transform(shapes.begin(), shapes.end(), std::back_inserter(ret_shapes), + [](const TensorShape &s) -> std::vector { return s.AsVector(); }); + return ret_shapes; } int64_t Dataset::GetNumClasses() { @@ -296,16 +335,31 @@ BatchDataset::BatchDataset(std::shared_ptr input, int32_t batch_size, b BucketBatchByLengthDataset::BucketBatchByLengthDataset( std::shared_ptr input, const std::vector> &column_names, const std::vector &bucket_boundaries, const std::vector &bucket_batch_sizes, - std::function element_length_function, - const std::map, std::pair>> &pad_info, - bool pad_to_bucket_boundary, bool drop_remainder) { + std::function element_length_function, + const std::map, std::pair, MSTensor>> &pad_info, bool pad_to_bucket_boundary, + bool drop_remainder) { std::shared_ptr c_func = nullptr; if (element_length_function != nullptr) { - c_func = std::make_shared(element_length_function); + c_func = std::make_shared(std::bind(FuncPtrConverter, element_length_function, std::placeholders::_1)); + } + + std::map, std::pair>> map; + for (auto const &p : pad_info) { + const MSTensor &t = p.second.second; + std::shared_ptr rt; + Status rc = Tensor::CreateFromMemory(TensorShape(t.Shape()), MSTypeToDEType(static_cast(t.DataType())), + (const uchar *)(t.Data().get()), t.DataSize(), &rt); + if (rc.IsError()) { + MS_LOG_ERROR << "Fail to create DETensor from MSTensor for pad_info: " << rc.ToString() << "."; + map.clear(); + break; + } + map.insert({p.first, {TensorShape(p.second.first), rt}}); } - auto ds = std::make_shared( - input->IRNode(), VectorCharToString(column_names), bucket_boundaries, bucket_batch_sizes, c_func, - PadInfoCharToString(pad_info), pad_to_bucket_boundary, drop_remainder); + + auto ds = std::make_shared(input->IRNode(), VectorCharToString(column_names), + bucket_boundaries, bucket_batch_sizes, c_func, + PadInfoCharToString(map), pad_to_bucket_boundary, drop_remainder); ir_node_ = std::static_pointer_cast(ds); } @@ -322,10 +376,10 @@ ConcatDataset::ConcatDataset(const std::vector> &datase ir_node_ = std::static_pointer_cast(ds); } -FilterDataset::FilterDataset(std::shared_ptr input, std::function predicate, +FilterDataset::FilterDataset(std::shared_ptr input, std::function predicate, const std::vector> &input_columns) { std::shared_ptr c_func = nullptr; - if (predicate) c_func = std::make_shared(predicate); + if (predicate) c_func = std::make_shared(std::bind(FuncPtrConverter, predicate, std::placeholders::_1)); auto ds = std::make_shared(input->IRNode(), c_func, VectorCharToString(input_columns)); ir_node_ = std::static_pointer_cast(ds); @@ -528,8 +582,9 @@ Status SchemaObj::Init() { } // Function to add a column to schema with a mstype de_type and known shape -Status SchemaObj::add_column_char(const std::vector &name, TypeId de_type, const std::vector &shape) { - DataType data_type = dataset::MSTypeToDEType(de_type); +Status SchemaObj::add_column_char(const std::vector &name, mindspore::DataType de_type, + const std::vector &shape) { + DataType data_type = dataset::MSTypeToDEType(static_cast(de_type)); return add_column_char(name, StringToChar(data_type.ToString()), shape); } @@ -550,8 +605,8 @@ Status SchemaObj::add_column_char(const std::vector &name, const std::vect } // Function to add a column to schema with a mstype de_type and without shape -Status SchemaObj::add_column_char(const std::vector &name, TypeId de_type) { - DataType data_type = dataset::MSTypeToDEType(de_type); +Status SchemaObj::add_column_char(const std::vector &name, mindspore::DataType de_type) { + DataType data_type = dataset::MSTypeToDEType(static_cast(de_type)); return add_column_char(name, StringToChar(data_type.ToString())); } diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc index 8596d18717..f56a7f38b0 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc @@ -34,12 +34,16 @@ PYBIND_REGISTER( THROW_IF_ERROR(schema->Init()); return schema; })) - .def("add_column", [](SchemaObj &self, std::string name, TypeId de_type, - std::vector shape) { THROW_IF_ERROR(self.add_column(name, de_type, shape)); }) + .def("add_column", + [](SchemaObj &self, std::string name, TypeId de_type, std::vector shape) { + THROW_IF_ERROR(self.add_column(name, static_cast(de_type), shape)); + }) .def("add_column", [](SchemaObj &self, std::string name, std::string de_type, std::vector shape) { THROW_IF_ERROR(self.add_column(name, de_type, shape)); }) .def("add_column", - [](SchemaObj &self, std::string name, TypeId de_type) { THROW_IF_ERROR(self.add_column(name, de_type)); }) + [](SchemaObj &self, std::string name, TypeId de_type) { + THROW_IF_ERROR(self.add_column(name, static_cast(de_type))); + }) .def("add_column", [](SchemaObj &self, std::string name, std::string de_type) { THROW_IF_ERROR(self.add_column(name, de_type)); }) .def("parse_columns", diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.cc b/mindspore/ccsrc/minddata/dataset/core/tensor.cc index 32edfa451c..d35e87bfda 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.cc @@ -24,6 +24,8 @@ #include #include +#include "minddata/dataset/include/type_id.h" + #include "utils/ms_utils.h" #include "minddata/dataset/core/constants.h" @@ -1040,6 +1042,10 @@ Status Tensor::SliceString(std::shared_ptr *out, const std::vector(in.DataType())), + (const uchar *)(in.Data().get()), in.DataSize(), out); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index d176053ddc..220db2ddba 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -117,6 +117,11 @@ class Tensor { return CreateFromMemory(in->shape(), in->type(), in->GetBuffer(), in->SizeInBytes(), out); } + /// Create a copy of the input tensor + /// \param[in] MSTensor to create DETensorFrom + /// \return Status + static Status CreateFromMSTensor(const MSTensor &in, TensorPtr *out); + #ifdef ENABLE_PYTHON /// Create a Tensor from a given py::array /// \param[in] arr py::array diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index 58720314c9..ff6e87e5dd 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -30,10 +30,12 @@ #include #include +#include + #include "include/api/dual_abi_helper.h" +#include "include/api/types.h" #include "minddata/dataset/include/iterator.h" #include "minddata/dataset/include/samplers.h" -#include "minddata/dataset/include/tensor.h" #include "minddata/dataset/include/text.h" #include "minddata/dataset/include/type_id.h" @@ -41,7 +43,6 @@ namespace mindspore { namespace dataset { class Tensor; -class TensorRow; class TensorShape; class TreeAdapter; class TreeGetters; @@ -101,11 +102,11 @@ class Dataset : public std::enable_shared_from_this { /// \brief Gets the output type /// \return a vector of DataType. If failed, return an empty vector - std::vector GetOutputTypes(); + std::vector GetOutputTypes(); /// \brief Gets the output shape /// \return a vector of TensorShape. If failed, return an empty vector - std::vector GetOutputShapes(); + std::vector> GetOutputShapes(); /// \brief Gets the batch size /// \return int64_t @@ -196,7 +197,7 @@ class Dataset : public std::enable_shared_from_this { /// 0 { std::shared_ptr BucketBatchByLength( const std::vector &column_names, const std::vector &bucket_boundaries, const std::vector &bucket_batch_sizes, - std::function element_length_function = nullptr, - const std::map>> &pad_info = {}, + std::function element_length_function = nullptr, + const std::map, MSTensor>> &pad_info = {}, bool pad_to_bucket_boundary = false, bool drop_remainder = false) { return std::make_shared( shared_from_this(), VectorStringToChar(column_names), bucket_boundaries, bucket_batch_sizes, @@ -276,7 +277,7 @@ class Dataset : public std::enable_shared_from_this { /// \param[in] predicate Function callable which returns a boolean value. If false then filter the element /// \param[in] input_columns List of names of the input columns to filter /// \return Shared pointer to the current FilterNode - std::shared_ptr Filter(std::function predicate, + std::shared_ptr Filter(std::function predicate, const std::vector &input_columns = {}) { return std::make_shared(shared_from_this(), predicate, VectorStringToChar(input_columns)); } @@ -481,35 +482,37 @@ class SchemaObj { /// \brief Add new column to the schema with unknown shape of rank 1 /// \param[in] name Name of the column. - /// \param[in] de_type Data type of the column(TypeId). + /// \param[in] ms_type Data type of the column(mindspore::DataType). /// \return Status code - Status add_column(const std::string &name, TypeId de_type) { return add_column_char(StringToChar(name), de_type); } + Status add_column(const std::string &name, mindspore::DataType ms_type) { + return add_column_char(StringToChar(name), ms_type); + } /// \brief Add new column to the schema with unknown shape of rank 1 /// \param[in] name Name of the column. - /// \param[in] de_type Data type of the column(std::string). + /// \param[in] ms_type Data type of the column(std::string). /// \param[in] shape Shape of the column. /// \return Status code - Status add_column(const std::string &name, const std::string &de_type) { - return add_column_char(StringToChar(name), StringToChar(de_type)); + Status add_column(const std::string &name, const std::string &ms_type) { + return add_column_char(StringToChar(name), StringToChar(ms_type)); } /// \brief Add new column to the schema /// \param[in] name Name of the column. - /// \param[in] de_type Data type of the column(TypeId). + /// \param[in] ms_type Data type of the column(mindspore::DataType). /// \param[in] shape Shape of the column. /// \return Status code - Status add_column(const std::string &name, TypeId de_type, const std::vector &shape) { - return add_column_char(StringToChar(name), de_type, shape); + Status add_column(const std::string &name, mindspore::DataType ms_type, const std::vector &shape) { + return add_column_char(StringToChar(name), ms_type, shape); } /// \brief Add new column to the schema /// \param[in] name Name of the column. - /// \param[in] de_type Data type of the column(std::string). + /// \param[in] ms_type Data type of the column(std::string). /// \param[in] shape Shape of the column. /// \return Status code - Status add_column(const std::string &name, const std::string &de_type, const std::vector &shape) { - return add_column_char(StringToChar(name), StringToChar(de_type), shape); + Status add_column(const std::string &name, const std::string &ms_type, const std::vector &shape) { + return add_column_char(StringToChar(name), StringToChar(ms_type), shape); } /// \brief Get a JSON string of the schema @@ -556,13 +559,13 @@ class SchemaObj { explicit SchemaObj(const std::vector &schema_file); // Char interface of add_column - Status add_column_char(const std::vector &name, TypeId de_type); + Status add_column_char(const std::vector &name, mindspore::DataType ms_type); - Status add_column_char(const std::vector &name, const std::vector &de_type); + Status add_column_char(const std::vector &name, const std::vector &ms_type); - Status add_column_char(const std::vector &name, TypeId de_type, const std::vector &shape); + Status add_column_char(const std::vector &name, mindspore::DataType ms_type, const std::vector &shape); - Status add_column_char(const std::vector &name, const std::vector &de_type, + Status add_column_char(const std::vector &name, const std::vector &ms_type, const std::vector &shape); // Char interface of to_json @@ -589,8 +592,8 @@ class BucketBatchByLengthDataset : public Dataset { BucketBatchByLengthDataset( std::shared_ptr input, const std::vector> &column_names, const std::vector &bucket_boundaries, const std::vector &bucket_batch_sizes, - std::function element_length_function = nullptr, - const std::map, std::pair>> &pad_info = {}, + std::function element_length_function = nullptr, + const std::map, std::pair, MSTensor>> &pad_info = {}, bool pad_to_bucket_boundary = false, bool drop_remainder = false); ~BucketBatchByLengthDataset() = default; }; @@ -603,7 +606,7 @@ class ConcatDataset : public Dataset { class FilterDataset : public Dataset { public: - FilterDataset(std::shared_ptr input, std::function predicate, + FilterDataset(std::shared_ptr input, std::function predicate, const std::vector> &input_columns); ~FilterDataset() = default; }; @@ -1530,12 +1533,8 @@ std::shared_ptr TFRecord(const std::vector &datase std::string schema_path = schema; if (!schema_path.empty()) { struct stat sb; - int rc = stat(common::SafeCStr(schema_path), &sb); - if (rc == -1 && errno != ENOENT) { - MS_LOG(WARNING) << "Unable to query the status of [" << schema_path << "]. Errno = " << errno << "."; - } + int rc = stat(schema_path.c_str(), &sb); if (rc != 0) { - MS_LOG(ERROR) << "TFRecordDataset: schema path [" << schema_path << "] is invalid or does not exist."; return nullptr; } } diff --git a/tests/ut/cpp/dataset/c_api_cache_test.cc b/tests/ut/cpp/dataset/c_api_cache_test.cc index 95d0d42690..b3f7a43d0e 100644 --- a/tests/ut/cpp/dataset/c_api_cache_test.cc +++ b/tests/ut/cpp/dataset/c_api_cache_test.cc @@ -210,7 +210,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) { // Create a CelebA Dataset, this folder_path has 4 records in it std::string folder_path = datasets_root_path_ + "/testCelebAData/"; - std::shared_ptr ds = CelebA(folder_path, "all", std::make_shared(false, 10), false, {}, some_cache); + std::shared_ptr ds = + CelebA(folder_path, "all", std::make_shared(false, 10), false, {}, some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -374,7 +375,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) { // Create a VOC Dataset, this folder_path has 9 records in it std::string folder_path = datasets_root_path_ + "/testVOC2012/"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared(), some_cache); + std::shared_ptr ds = + VOC(folder_path, "Detection", "train", {}, false, std::make_shared(), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -417,7 +419,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheAlbumCApi) { std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::vector column_names = {"image", "label", "id"}; // Create a Album Dataset, 7 records in it - std::shared_ptr ds = Album(folder_path, schema_file, column_names, false, std::make_shared(), some_cache); + std::shared_ptr ds = + Album(folder_path, schema_file, column_names, false, std::make_shared(), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -456,8 +459,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheRandomDataCApi) { // Create a RandomDataset std::shared_ptr schema = Schema(); - schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2}); - schema->add_column("label", mindspore::TypeId::kNumberTypeUInt8, {1}); + + schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}); + schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}); std::shared_ptr ds = RandomData(4, schema, {}, some_cache); EXPECT_NE(ds, nullptr); @@ -826,7 +830,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) { // Since sampler does not influence the data in the source, these two pipelines can share a common cache. std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(), {}, {}, some_cache); + std::shared_ptr ds2 = + ImageFolder(folder_path, true, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds2, nullptr); // Create and launch the Execution Tree for ds1 diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index a8fd2daffb..6609ab3d61 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -14,6 +14,7 @@ * limitations under the License. */ #include "common/common.h" + #include "minddata/dataset/include/datasets.h" using namespace mindspore::dataset; @@ -134,8 +135,8 @@ TEST_F(MindDataTestPipeline, TestCifar10Getters) { EXPECT_NE(ds, nullptr); EXPECT_EQ(ds->GetDatasetSize(), 10000); - std::vector types = ds->GetOutputTypes(); - std::vector shapes = ds->GetOutputShapes(); + std::vector types = ToDETypes(ds->GetOutputTypes()); + std::vector shapes = ToTensorShapeVec(ds->GetOutputShapes()); std::vector column_names = {"image", "label"}; int64_t num_classes = ds->GetNumClasses(); EXPECT_EQ(types.size(), 2); @@ -149,14 +150,14 @@ TEST_F(MindDataTestPipeline, TestCifar10Getters) { EXPECT_EQ(ds->GetRepeatCount(), 1); EXPECT_EQ(ds->GetDatasetSize(), 10000); - EXPECT_EQ(ds->GetOutputTypes(), types); - EXPECT_EQ(ds->GetOutputShapes(), shapes); + EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types); + EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes); EXPECT_EQ(ds->GetNumClasses(), -1); EXPECT_EQ(ds->GetColumnNames(), column_names); EXPECT_EQ(ds->GetDatasetSize(), 10000); - EXPECT_EQ(ds->GetOutputTypes(), types); - EXPECT_EQ(ds->GetOutputShapes(), shapes); + EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types); + EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes); EXPECT_EQ(ds->GetBatchSize(), 1); EXPECT_EQ(ds->GetRepeatCount(), 1); EXPECT_EQ(ds->GetNumClasses(), -1); @@ -207,8 +208,8 @@ TEST_F(MindDataTestPipeline, TestCifar100Getters) { EXPECT_NE(ds, nullptr); std::vector column_names = {"image", "coarse_label", "fine_label"}; - std::vector types = ds->GetOutputTypes(); - std::vector shapes = ds->GetOutputShapes(); + std::vector types = ToDETypes(ds->GetOutputTypes()); + std::vector shapes = ToTensorShapeVec(ds->GetOutputShapes()); int64_t num_classes = ds->GetNumClasses(); EXPECT_EQ(types.size(), 3); diff --git a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc index 73ec59aaa5..ca6bbfd1f1 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc @@ -141,7 +141,8 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared(0, 4)); + std::shared_ptr ds = + VOC(folder_path, "Detection", "train", {}, false, std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -198,7 +199,7 @@ TEST_F(MindDataTestPipeline, TestIteratorNumEpoch) { std::shared_ptr schema = Schema(); int32_t random_data_num_row = 2; int32_t num_epochs = 3; - ASSERT_OK(schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2})); + ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2})); std::shared_ptr ds = RandomData(random_data_num_row, schema)->SetNumWorkers(1); std::shared_ptr iter = ds->CreateIterator({}, num_epochs); @@ -228,7 +229,7 @@ TEST_F(MindDataTestPipeline, TestIteratorNumEpochFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorNumEpochFail."; std::shared_ptr schema = Schema(); - ASSERT_OK(schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2})); + ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2})); std::shared_ptr ds = RandomData(3, schema)->SetNumWorkers(1); // expect nullptr due to incorrect num_epochs value. EXPECT_EQ(ds->CreateIterator({}, 0), nullptr); diff --git a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc index c00a4a02e5..98a3663188 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc @@ -15,6 +15,8 @@ */ #include "common/common.h" #include "minddata/dataset/include/datasets.h" +#include "minddata/dataset/core/tensor.h" + using namespace mindspore::dataset; using mindspore::dataset::Tensor; @@ -283,7 +285,8 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) { // Pass a list of mindrecord file name, files in list will be read directly but not search for related files std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::vector file_list = {file_path1}; - std::shared_ptr ds = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); + std::shared_ptr ds = + MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds, nullptr); // Create a Skip operation on ds, skip original data in mindrecord and get padded samples @@ -332,11 +335,12 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) { // Pass a list of mindrecord file name, files in list will be read directly but not search for related files std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::vector file_list = {file_path1}; - std::shared_ptr ds = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); + std::shared_ptr ds = + MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds, nullptr); - std::vector types = ds->GetOutputTypes(); - std::vector shapes = ds->GetOutputShapes(); + std::vector types = ToDETypes(ds->GetOutputTypes()); + std::vector shapes = ToTensorShapeVec(ds->GetOutputShapes()); std::vector column_names = {"file_name", "label"}; EXPECT_EQ(types.size(), 2); EXPECT_EQ(types[0].ToString(), "string"); @@ -400,12 +404,14 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) { // Pass a list of mindrecord file name, files in list will be read directly but not search for related files std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::vector file_list = {file_path1}; - std::shared_ptr ds1 = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); + std::shared_ptr ds1 = + MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds1, nullptr); ds1 = ds1->Skip(5); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); + std::shared_ptr ds2 = + MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds2, nullptr); ds2 = ds2->Skip(5); EXPECT_NE(ds2, nullptr); @@ -598,7 +604,8 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { // Create a MindData Dataset std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds4 = MindData(file_path4, {"file_name", "label"}, std::make_shared(), pad2, 1); + std::shared_ptr ds4 = + MindData(file_path4, {"file_name", "label"}, std::make_shared(), pad2, 1); EXPECT_NE(ds4, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index 85e054ca47..fa907163fc 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -14,6 +14,7 @@ * limitations under the License. */ #include "common/common.h" +#include "include/api/types.h" #include "minddata/dataset/core/tensor_row.h" #include "minddata/dataset/engine/ir/datasetops/dataset_node.h" #include "minddata/dataset/include/datasets.h" @@ -26,19 +27,39 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; -mindspore::dataset::TensorRow BucketBatchTestFunction(mindspore::dataset::TensorRow input) { +TensorRow VecToRow(const MSTensorVec &v) { + TensorRow row; + for (const mindspore::MSTensor &t : v) { + std::shared_ptr rt; + Status rc = + Tensor::CreateFromMemory(TensorShape(t.Shape()), MSTypeToDEType(static_cast(t.DataType())), + (const uchar *)(t.Data().get()), t.DataSize(), &rt); + row.emplace_back(rt); + } + return row; +} +MSTensorVec RowToVec(const TensorRow &v) { + MSTensorVec rv; // std::make_shared(de_tensor) + std::transform(v.begin(), v.end(), std::back_inserter(rv), [](std::shared_ptr t) -> mindspore::MSTensor { + return mindspore::MSTensor(std::make_shared(t)); + }); + return rv; +} + +MSTensorVec BucketBatchTestFunction(MSTensorVec input) { mindspore::dataset::TensorRow output; std::shared_ptr out; Tensor::CreateEmpty(mindspore::dataset::TensorShape({1}), mindspore::dataset::DataType(mindspore::dataset::DataType::Type::DE_INT32), &out); out->SetItemAt({0}, 2); output.push_back(out); - return output; + return RowToVec(output); } -TensorRow Predicate1(TensorRow input) { +MSTensorVec Predicate1(MSTensorVec in) { // Return true if input is equal to 3 uint64_t input_value; + TensorRow input = VecToRow(in); input.at(0)->GetItemAt(&input_value, {0}); bool result = (input_value == 3); @@ -50,13 +71,14 @@ TensorRow Predicate1(TensorRow input) { out->SetItemAt({0}, result); output.push_back(out); - return output; + return RowToVec(output); } -TensorRow Predicate2(TensorRow input) { +MSTensorVec Predicate2(MSTensorVec in) { // Return true if label is more than 1 // The index of label in input is 1 uint64_t input_value; + TensorRow input = VecToRow(in); input.at(1)->GetItemAt(&input_value, {0}); bool result = (input_value > 1); @@ -68,7 +90,7 @@ TensorRow Predicate2(TensorRow input) { out->SetItemAt({0}, result); output.push_back(out); - return output; + return RowToVec(output); } TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { @@ -158,7 +180,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) { EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds - std::map>> pad_info; + std::map, mindspore::MSTensor>> pad_info = {}; ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, 2, 3}, &BucketBatchTestFunction, pad_info, true, true); EXPECT_NE(ds, nullptr); @@ -682,7 +704,7 @@ TEST_F(MindDataTestPipeline, TestFilterFail1) { std::shared_ptr ds = TFRecord({data_file}, schema_file, {"image", "label"}, 0, ShuffleMode::kFalse); EXPECT_NE(ds, nullptr); - std::function predicate_null = nullptr; + std::function predicate_null = nullptr; // Create a Filter operation on ds ds = ds->Filter(predicate_null); @@ -998,22 +1020,22 @@ TEST_F(MindDataTestPipeline, TestMapDuplicateColumnFail) { } TEST_F(MindDataTestPipeline, TestMapNullOperation) { -MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMapNullOperation."; - -// Create an ImageFolder Dataset -std::string folder_path = datasets_root_path_ + "/testPK/data/"; -std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); -EXPECT_NE(ds, nullptr); - -// Create a Map operation on ds -std::shared_ptr operation = nullptr; -auto ds1 = ds->Map({operation}, {"image"}, {}, {}); -EXPECT_NE(ds1, nullptr); - -// Create an iterator over the result of the above dataset -std::shared_ptr iter1 = ds1->CreateIterator(); -// Expect failure: Operation is nullptr -EXPECT_EQ(iter1, nullptr); + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMapNullOperation."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Map operation on ds + std::shared_ptr operation = nullptr; + auto ds1 = ds->Map({operation}, {"image"}, {}, {}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter1 = ds1->CreateIterator(); + // Expect failure: Operation is nullptr + EXPECT_EQ(iter1, nullptr); } TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { diff --git a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc index ee4eccbc65..6b3b1e7fc6 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc @@ -30,8 +30,8 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic1) { // Create a RandomDataset std::shared_ptr schema = Schema(); - schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2}); - schema->add_column("label", mindspore::TypeId::kNumberTypeUInt8, {1}); + schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}); + schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}); std::shared_ptr ds = RandomData(50, schema); EXPECT_NE(ds, nullptr); @@ -74,8 +74,8 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasicWithPipeline) { // Create two RandomDataset std::shared_ptr schema = Schema(); - schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2}); - schema->add_column("label", mindspore::TypeId::kNumberTypeUInt8, {1}); + schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}); + schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}); std::shared_ptr ds1 = RandomData(50, schema); std::shared_ptr ds2 = RandomData(50, schema); EXPECT_NE(ds1, nullptr); @@ -132,8 +132,8 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetGetters) { // Create a RandomDataset std::shared_ptr schema = Schema(); - schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2}); - schema->add_column("label", mindspore::TypeId::kNumberTypeUInt8, {1}); + schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}); + schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}); std::shared_ptr ds = RandomData(50, schema); EXPECT_NE(ds, nullptr); @@ -480,8 +480,8 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetDuplicateColumnName) { // Create a RandomDataset std::shared_ptr schema = Schema(); - schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2}); - schema->add_column("label", mindspore::TypeId::kNumberTypeUInt8, {1}); + schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}); + schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}); std::shared_ptr ds = RandomData(50, schema, {"image", "image"}); // Expect failure: duplicate column names EXPECT_EQ(ds->CreateIterator(), nullptr); diff --git a/tests/ut/cpp/dataset/c_api_text_test.cc b/tests/ut/cpp/dataset/c_api_text_test.cc index 5a1093fc7d..053c94e396 100644 --- a/tests/ut/cpp/dataset/c_api_text_test.cc +++ b/tests/ut/cpp/dataset/c_api_text_test.cc @@ -1651,8 +1651,8 @@ TEST_F(MindDataTestPipeline, TestTruncateSequencePairSuccess1) { // Create a RandomDataset which has column names "col1" and "col2" std::shared_ptr schema = Schema(); - schema->add_column("col1", mindspore::TypeId::kNumberTypeInt16, {5}); - schema->add_column("col2", mindspore::TypeId::kNumberTypeInt32, {3}); + schema->add_column("col1", mindspore::DataType::kNumberTypeInt16, {5}); + schema->add_column("col2", mindspore::DataType::kNumberTypeInt32, {3}); std::shared_ptr ds = RandomData(3, schema); EXPECT_NE(ds, nullptr); @@ -1719,8 +1719,8 @@ TEST_F(MindDataTestPipeline, TestTruncateSequencePairSuccess2) { // Create a RandomDataset which has column names "col1" and "col2" std::shared_ptr schema = Schema(); - schema->add_column("col1", mindspore::TypeId::kNumberTypeInt32, {4}); - schema->add_column("col2", mindspore::TypeId::kNumberTypeInt64, {4}); + schema->add_column("col1", mindspore::DataType::kNumberTypeInt32, {4}); + schema->add_column("col2", mindspore::DataType::kNumberTypeInt64, {4}); std::shared_ptr ds = RandomData(4, schema); EXPECT_NE(ds, nullptr); @@ -1780,8 +1780,8 @@ TEST_F(MindDataTestPipeline, TestTruncateSequencePairFail) { // Create a RandomDataset which has column names "col1" and "col2" std::shared_ptr schema = Schema(); - schema->add_column("col1", mindspore::TypeId::kNumberTypeInt8, {3}); - schema->add_column("col2", mindspore::TypeId::kNumberTypeInt8, {3}); + schema->add_column("col1", mindspore::DataType::kNumberTypeInt8, {3}); + schema->add_column("col2", mindspore::DataType::kNumberTypeInt8, {3}); std::shared_ptr ds = RandomData(3, schema); EXPECT_NE(ds, nullptr); diff --git a/tests/ut/cpp/dataset/common/common.cc b/tests/ut/cpp/dataset/common/common.cc index 7c55575680..ed5f83ffb2 100644 --- a/tests/ut/cpp/dataset/common/common.cc +++ b/tests/ut/cpp/dataset/common/common.cc @@ -22,18 +22,31 @@ extern "C" { #endif #endif - void DatasetOpTesting::SetUp() { std::string install_home = "data/dataset"; datasets_root_path_ = install_home; mindrecord_root_path_ = "data/mindrecord"; } +std::vector DatasetOpTesting::ToTensorShapeVec( + const std::vector> &v) { + std::vector ret_v; + std::transform(v.begin(), v.end(), std::back_inserter(ret_v), + [](const auto &s) { return mindspore::dataset::TensorShape(s); }); + return ret_v; +} + +std::vector DatasetOpTesting::ToDETypes(const std::vector &t) { + std::vector ret_t; + std::transform(t.begin(), t.end(), std::back_inserter(ret_t), [](const mindspore::DataType &t) { + return mindspore::dataset::MSTypeToDEType(static_cast(t)); + }); + return ret_t; +} + #ifdef __cplusplus #if __cplusplus } #endif #endif } // namespace UT - - diff --git a/tests/ut/cpp/dataset/common/common.h b/tests/ut/cpp/dataset/common/common.h index 474a2c41ee..78ab0ededa 100644 --- a/tests/ut/cpp/dataset/common/common.h +++ b/tests/ut/cpp/dataset/common/common.h @@ -18,6 +18,9 @@ #include "gtest/gtest.h" #include "include/api/status.h" +#include "include/api/types.h" +#include "minddata/dataset/core/tensor_shape.h" +#include "minddata/dataset/include/type_id.h" #include "utils/log_adapter.h" using mindspore::Status; @@ -62,19 +65,19 @@ using mindspore::StatusCode; namespace UT { class Common : public testing::Test { public: - // every TEST_F macro will enter one - virtual void SetUp(); + // every TEST_F macro will enter one + virtual void SetUp(); - virtual void TearDown(); + virtual void TearDown(); }; class DatasetOpTesting : public Common { public: - std::string datasets_root_path_; - std::string mindrecord_root_path_; - void SetUp() override; - + std::vector ToTensorShapeVec(const std::vector> &v); + std::vector ToDETypes(const std::vector &t); + std::string datasets_root_path_; + std::string mindrecord_root_path_; + void SetUp() override; }; } // namespace UT #endif // TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ - diff --git a/tests/ut/cpp/dataset/ir_callback_test.cc b/tests/ut/cpp/dataset/ir_callback_test.cc index ab4dd980e9..229918f8e9 100644 --- a/tests/ut/cpp/dataset/ir_callback_test.cc +++ b/tests/ut/cpp/dataset/ir_callback_test.cc @@ -329,7 +329,7 @@ TEST_F(MindDataTestCallback, TestCAPICallback) { std::shared_ptr cb1 = tst_cb; // Create a RandomDataset. Use random_data to avoid I/O std::shared_ptr schema = Schema(); - ASSERT_OK(schema->add_column("label", mindspore::TypeId::kNumberTypeUInt32, {})); + ASSERT_OK(schema->add_column("label", mindspore::DataType::kNumberTypeUInt32, {})); std::shared_ptr ds = RandomData(44, schema); ASSERT_NE(ds, nullptr); ds = ds->Map({std::make_shared("uint64")}, {"label"}, {}, {}, nullptr, {cb1});