diff --git a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc index 541ed9b47a..cc00eb794d 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc @@ -56,6 +56,61 @@ TEST_F(MindDataTestPipeline, TestAlbumBasic) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasic."; + + std::string folder_path = datasets_root_path_ + "/testAlbum/images"; + std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; + std::vector column_names = {"image", "label", "id"}; + + // Create two Album Dataset + std::shared_ptr ds1 = Album(folder_path, schema_file, column_names); + std::shared_ptr ds2 = Album(folder_path, schema_file, column_names); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 2; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 3; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"image"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 35); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestAlbumGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumGetters."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index 4045f840f4..77099bfd3e 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -59,6 +59,61 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10Dataset."; + + // Create two Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds1 = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 1; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 1; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"image", "label"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 20); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestCifar10GetDatasetSize) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10GetDatasetSize."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc index 7cce11d7bb..3da344a8f7 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc @@ -147,6 +147,62 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestCLUEDatasetBasicWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetBasic."; + + // Create two CLUEFile Dataset, with single CLUE file + std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json"; + std::string task = "AFQMC"; + std::string usage = "train"; + std::shared_ptr ds1 = CLUE({clue_file}, task, usage, 2); + std::shared_ptr ds2 = CLUE({clue_file}, task, usage, 2); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 2; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 3; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"sentence1"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + EXPECT_NE(row.find("sentence1"), row.end()); + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["sentence1"]; + MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + i++; + iter->GetNextRow(&row); + } + + // Expect 10 samples + EXPECT_EQ(i, 10); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestCLUEGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEGetters."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc index 0756eb52c0..ac10d45f04 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc @@ -61,6 +61,63 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestCocoDefaultWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDefault."; + // Create two Coco Dataset + std::string folder_path = datasets_root_path_ + "/testCOCO/train"; + std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; + + std::shared_ptr ds1 = Coco(folder_path, annotation_file); + std::shared_ptr ds2 = Coco(folder_path, annotation_file); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 2; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 3; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"image", "bbox", "category_id"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + auto image = row["image"]; + auto bbox = row["bbox"]; + auto category_id = row["category_id"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape(); + MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape(); + iter->GetNextRow(&row); + i++; + } + + EXPECT_EQ(i, 30); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestCocoGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoGetters."; // Create a Coco Dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index f67e8bc73f..8bd775cabb 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -54,6 +54,58 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasic."; + + std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; + // Create two Manifest Dataset + std::shared_ptr ds1 = Manifest(file_path); + std::shared_ptr ds2 = Manifest(file_path); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 2; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 3; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"image"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 10); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestManifestGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestGetters."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc index 0c4c9248b4..f8f60b7b29 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc @@ -69,6 +69,65 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic1) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestRandomDatasetBasicWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic1."; + + // Create two RandomDataset + std::shared_ptr schema = Schema(); + schema->add_column("image", mindspore::TypeId::kNumberTypeUInt8, {2}); + schema->add_column("label", mindspore::TypeId::kNumberTypeUInt8, {1}); + std::shared_ptr ds1 = RandomData(50, schema); + std::shared_ptr ds2 = RandomData(50, schema); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 2; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 2; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"image", "label"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + // Check if RandomDataOp read correct columns + uint64_t i = 0; + while (row.size() != 0) { + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Tensor label shape: " << label->shape(); + + iter->GetNextRow(&row); + i++; + } + + EXPECT_EQ(i, 200); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestRandomDatasetGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetGetters.";