Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

dataset: Reinstate INFO logging and data verification - part 2 

for image/vision datasets
tags/v1.2.0-rc1
Cathy Wong 5 years ago
parent
c13af64986
commit
95bbaaf9f6
10 changed files with 294 additions and 234 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +29
    -29
      tests/ut/cpp/dataset/c_api_cache_test.cc
  2. +137
    -101
      tests/ut/cpp/dataset/c_api_dataset_coco_test.cc
  3. +27
    -18
      tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc
  4. +8
    -8
      tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc
  5. +27
    -25
      tests/ut/cpp/dataset/c_api_dataset_voc_test.cc
  6. +18
    -16
      tests/ut/cpp/dataset/c_api_datasets_test.cc
  7. +2
    -2
      tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc
  8. +32
    -0
      tests/ut/cpp/dataset/common/common.cc
  9. +13
    -3
      tests/ut/cpp/dataset/common/common.h
  10. +1
    -32
      tests/ut/cpp/dataset/execute_test.cc

+ 29
- 29
tests/ut/cpp/dataset/c_api_cache_test.cc View File

@@ -105,8 +105,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheImageFolderCApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -148,8 +148,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCocoCApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -189,8 +189,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheMnistCApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -231,8 +231,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -272,8 +272,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheManifestCApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -313,8 +313,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar10CApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -354,8 +354,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar100CApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -396,8 +396,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -523,8 +523,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheTFRecordCApi1) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -574,8 +574,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheTFRecordCApi2) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -621,8 +621,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheTFRecordCApi3) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

@@ -789,8 +789,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter1->GetNextRow(&row);
}
EXPECT_EQ(i, 2);
@@ -806,8 +806,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) {
i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter2->GetNextRow(&row);
}
EXPECT_EQ(i, 2);
@@ -844,7 +844,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
auto image = row["image"];
iter1->GetNextRow(&row);
}
EXPECT_EQ(i, 2);
@@ -860,7 +860,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) {
i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
auto image = row["image"];
iter2->GetNextRow(&row);
}
EXPECT_EQ(i, 2);
@@ -894,7 +894,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
auto image = row["image"];
iter1->GetNextRow(&row);
}
EXPECT_EQ(i, 2);


+ 137
- 101
tests/ut/cpp/dataset/c_api_dataset_coco_test.cc View File

@@ -138,7 +138,8 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";

std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, std::make_shared<SequentialSampler>(0, 6));
std::shared_ptr<Dataset> ds =
Coco(folder_path, annotation_file, "Detection", false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr);

// Create an iterator over the result of the above dataset
@@ -150,30 +151,37 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// std::string expect_file[] = {"000000391895", "000000318219", "000000554625",
// "000000574769", "000000060623", "000000309022"};
// std::vector<std::vector<float>> expect_bbox_vector = {{10.0, 10.0, 10.0, 10.0, 70.0, 70.0, 70.0, 70.0},
// {20.0, 20.0, 20.0, 20.0, 80.0, 80.0, 80.0, 80.0},
// {30.0, 30.0, 30.0, 30.0},
// {40.0, 40.0, 40.0, 40.0},
// {50.0, 50.0, 50.0, 50.0},
// {60.0, 60.0, 60.0, 60.0}};
// std::vector<std::vector<uint32_t>> expect_catagoryid_list = {{1, 7}, {2, 8}, {3}, {4}, {5}, {6}};
std::string expect_file[] = {"000000391895", "000000318219", "000000554625",
"000000574769", "000000060623", "000000309022"};
std::vector<std::vector<float>> expect_bbox_vector = {{10.0, 10.0, 10.0, 10.0, 70.0, 70.0, 70.0, 70.0},
{20.0, 20.0, 20.0, 20.0, 80.0, 80.0, 80.0, 80.0},
{30.0, 30.0, 30.0, 30.0},
{40.0, 40.0, 40.0, 40.0},
{50.0, 50.0, 50.0, 50.0},
{60.0, 60.0, 60.0, 60.0}};
std::vector<std::vector<uint32_t>> expect_catagoryid_list = {{1, 7}, {2, 8}, {3}, {4}, {5}, {6}};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto bbox = row["bbox"];
// auto category_id = row["category_id"];
// mindspore::MSTensor expect_image;
// Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
// EXPECT_EQ(*image, *expect_image);
// mindspore::MSTensor expect_bbox;
// dsize_t bbox_num = static_cast<dsize_t>(expect_bbox_vector[i].size() / 4);
// Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape({bbox_num, 4}), &expect_bbox);
// EXPECT_EQ(*bbox, *expect_bbox);
// mindspore::MSTensor expect_categoryid;
// Tensor::CreateFromVector(expect_catagoryid_list[i], TensorShape({bbox_num, 1}), &expect_categoryid);
// EXPECT_EQ(*category_id, *expect_categoryid);
auto image = row["image"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];

mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/" + expect_file[i] + ".jpg");
EXPECT_MSTENSOR_EQ(image, expect_image);

std::shared_ptr<Tensor> de_expect_bbox;
dsize_t bbox_num = static_cast<dsize_t>(expect_bbox_vector[i].size() / 4);
ASSERT_OK(Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape({bbox_num, 4}), &de_expect_bbox));
mindspore::MSTensor expect_bbox =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_bbox));
EXPECT_MSTENSOR_EQ(bbox, expect_bbox);

std::shared_ptr<Tensor> de_expect_categoryid;
ASSERT_OK(Tensor::CreateFromVector(expect_catagoryid_list[i], TensorShape({bbox_num, 1}), &de_expect_categoryid));
mindspore::MSTensor expect_categoryid =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_categoryid));
EXPECT_MSTENSOR_EQ(category_id, expect_categoryid);

iter->GetNextRow(&row);
i++;
}
@@ -220,7 +228,8 @@ TEST_F(MindDataTestPipeline, TestCocoKeypoint) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json";

std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, std::make_shared<SequentialSampler>(0, 2));
std::shared_ptr<Dataset> ds =
Coco(folder_path, annotation_file, "Keypoint", false, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr);

// Create an iterator over the result of the above dataset
@@ -232,33 +241,39 @@ TEST_F(MindDataTestPipeline, TestCocoKeypoint) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// std::string expect_file[] = {"000000391895", "000000318219"};
// std::vector<std::vector<float>> expect_keypoint_vector = {
// {368.0, 61.0, 1.0, 369.0, 52.0, 2.0, 0.0, 0.0, 0.0, 382.0, 48.0, 2.0, 0.0, 0.0, 0.0,
// 368.0, 84.0, 2.0,
// 435.0, 81.0, 2.0, 362.0, 125.0, 2.0, 446.0, 125.0, 2.0, 360.0, 153.0, 2.0, 0.0, 0.0, 0.0, 397.0,
// 167.0, 1.0, 439.0, 166.0, 1.0, 369.0, 193.0, 2.0, 461.0, 234.0, 2.0, 361.0, 246.0, 2.0, 474.0, 287.0, 2.0},
// {244.0, 139.0, 2.0, 0.0, 0.0, 0.0, 226.0, 118.0, 2.0, 0.0, 0.0, 0.0, 154.0, 159.0, 2.0, 143.0,
// 261.0, 2.0,
// 135.0, 312.0, 2.0, 271.0, 423.0, 2.0, 184.0, 530.0, 2.0, 261.0, 280.0, 2.0, 347.0, 592.0, 2.0, 0.0, 0.0, 0.0,
// 123.0, 596.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}};
// std::vector<std::vector<dsize_t>> expect_size = {{1, 51}, {1, 51}};
// std::vector<std::vector<uint32_t>> expect_num_keypoints_list = {{14}, {10}};
std::string expect_file[] = {"000000391895", "000000318219"};
std::vector<std::vector<float>> expect_keypoint_vector = {
{368.0, 61.0, 1.0, 369.0, 52.0, 2.0, 0.0, 0.0, 0.0, 382.0, 48.0, 2.0, 0.0, 0.0, 0.0, 368.0, 84.0, 2.0,
435.0, 81.0, 2.0, 362.0, 125.0, 2.0, 446.0, 125.0, 2.0, 360.0, 153.0, 2.0, 0.0, 0.0, 0.0, 397.0, 167.0, 1.0,
439.0, 166.0, 1.0, 369.0, 193.0, 2.0, 461.0, 234.0, 2.0, 361.0, 246.0, 2.0, 474.0, 287.0, 2.0},
{244.0, 139.0, 2.0, 0.0, 0.0, 0.0, 226.0, 118.0, 2.0, 0.0, 0.0, 0.0, 154.0, 159.0, 2.0, 143.0, 261.0, 2.0,
135.0, 312.0, 2.0, 271.0, 423.0, 2.0, 184.0, 530.0, 2.0, 261.0, 280.0, 2.0, 347.0, 592.0, 2.0, 0.0, 0.0, 0.0,
123.0, 596.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}};
std::vector<std::vector<dsize_t>> expect_size = {{1, 51}, {1, 51}};
std::vector<std::vector<uint32_t>> expect_num_keypoints_list = {{14}, {10}};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto keypoints = row["keypoints"];
// auto num_keypoints = row["num_keypoints"];
// mindspore::MSTensor expect_image;
// Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
// EXPECT_EQ(*image, *expect_image);
// mindspore::MSTensor expect_keypoints;
// dsize_t keypoints_size = expect_size[i][0];
// Tensor::CreateFromVector(expect_keypoint_vector[i], TensorShape(expect_size[i]), &expect_keypoints);
// EXPECT_EQ(*keypoints, *expect_keypoints);
// mindspore::MSTensor expect_num_keypoints;
// Tensor::CreateFromVector(expect_num_keypoints_list[i], TensorShape({keypoints_size, 1}), &expect_num_keypoints);
// EXPECT_EQ(*num_keypoints, *expect_num_keypoints);
auto image = row["image"];
auto keypoints = row["keypoints"];
auto num_keypoints = row["num_keypoints"];

mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/" + expect_file[i] + ".jpg");
EXPECT_MSTENSOR_EQ(image, expect_image);

std::shared_ptr<Tensor> de_expect_keypoints;
dsize_t keypoints_size = expect_size[i][0];
ASSERT_OK(Tensor::CreateFromVector(expect_keypoint_vector[i], TensorShape(expect_size[i]), &de_expect_keypoints));
mindspore::MSTensor expect_keypoints =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_keypoints));
EXPECT_MSTENSOR_EQ(keypoints, expect_keypoints);

std::shared_ptr<Tensor> de_expect_num_keypoints;
ASSERT_OK(Tensor::CreateFromVector(expect_num_keypoints_list[i], TensorShape({keypoints_size, 1}),
&de_expect_num_keypoints));
mindspore::MSTensor expect_num_keypoints =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_num_keypoints));
EXPECT_MSTENSOR_EQ(num_keypoints, expect_num_keypoints);

iter->GetNextRow(&row);
i++;
}
@@ -275,7 +290,8 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json";

std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
std::shared_ptr<Dataset> ds =
Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr);

// Create an iterator over the result of the above dataset
@@ -287,36 +303,50 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// std::string expect_file[] = {"000000391895", "000000574769"};
// std::vector<std::vector<float>> expect_bbox_vector = {{472, 173, 36, 48, 340, 22, 154, 301, 486, 183, 30, 35},
// {103, 133, 229, 422, 243, 175, 93, 164}};
// std::vector<std::vector<uint32_t>> expect_categoryid_vector = {{1, 1, 2}, {1, 3}};
// std::vector<std::vector<uint32_t>> expect_iscrowd_vector = {{0, 0, 0}, {0, 0}};
// std::vector<std::vector<uint32_t>> expect_area_vector = {{705, 14062, 626}, {43102, 6079}};
// std::vector<std::vector<dsize_t>> expect_size = {{3, 4}, {2, 4}};
std::string expect_file[] = {"000000391895", "000000574769"};
std::vector<std::vector<float>> expect_bbox_vector = {{472, 173, 36, 48, 340, 22, 154, 301, 486, 183, 30, 35},
{103, 133, 229, 422, 243, 175, 93, 164}};
std::vector<std::vector<uint32_t>> expect_categoryid_vector = {{1, 1, 2}, {1, 3}};
std::vector<std::vector<uint32_t>> expect_iscrowd_vector = {{0, 0, 0}, {0, 0}};
std::vector<std::vector<uint32_t>> expect_area_vector = {{705, 14062, 626}, {43102, 6079}};
std::vector<std::vector<dsize_t>> expect_size = {{3, 4}, {2, 4}};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto bbox = row["bbox"];
// auto category_id = row["category_id"];
// auto iscrowd = row["iscrowd"];
// auto area = row["area"];
// mindspore::MSTensor expect_image;
// Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
// EXPECT_EQ(*image, *expect_image);
// mindspore::MSTensor expect_bbox;
// dsize_t bbox_size = expect_size[i][0];
// Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape(expect_size[i]), &expect_bbox);
// EXPECT_EQ(*bbox, *expect_bbox);
// mindspore::MSTensor expect_categoryid;
// Tensor::CreateFromVector(expect_categoryid_vector[i], TensorShape({bbox_size, 1}), &expect_categoryid);
// EXPECT_EQ(*category_id, *expect_categoryid);
// mindspore::MSTensor expect_iscrowd;
// Tensor::CreateFromVector(expect_iscrowd_vector[i], TensorShape({bbox_size, 1}), &expect_iscrowd);
// EXPECT_EQ(*iscrowd, *expect_iscrowd);
// mindspore::MSTensor expect_area;
// Tensor::CreateFromVector(expect_area_vector[i], TensorShape({bbox_size, 1}), &expect_area);
// EXPECT_EQ(*area, *expect_area);
auto image = row["image"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];
auto iscrowd = row["iscrowd"];
auto area = row["area"];

mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/" + expect_file[i] + ".jpg");
EXPECT_MSTENSOR_EQ(image, expect_image);

std::shared_ptr<Tensor> de_expect_bbox;
dsize_t bbox_size = expect_size[i][0];
ASSERT_OK(Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape(expect_size[i]), &de_expect_bbox));
mindspore::MSTensor expect_bbox =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_bbox));
EXPECT_MSTENSOR_EQ(bbox, expect_bbox);

std::shared_ptr<Tensor> de_expect_categoryid;
ASSERT_OK(
Tensor::CreateFromVector(expect_categoryid_vector[i], TensorShape({bbox_size, 1}), &de_expect_categoryid));
mindspore::MSTensor expect_categoryid =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_categoryid));
EXPECT_MSTENSOR_EQ(category_id, expect_categoryid);

std::shared_ptr<Tensor> de_expect_iscrowd;
ASSERT_OK(Tensor::CreateFromVector(expect_iscrowd_vector[i], TensorShape({bbox_size, 1}), &de_expect_iscrowd));
mindspore::MSTensor expect_iscrowd =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_iscrowd));
EXPECT_MSTENSOR_EQ(iscrowd, expect_iscrowd);

std::shared_ptr<Tensor> de_expect_area;
ASSERT_OK(Tensor::CreateFromVector(expect_area_vector[i], TensorShape({bbox_size, 1}), &de_expect_area));
mindspore::MSTensor expect_area =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_area));
EXPECT_MSTENSOR_EQ(area, expect_area);

iter->GetNextRow(&row);
i++;
}
@@ -333,7 +363,8 @@ TEST_F(MindDataTestPipeline, TestCocoPanopticGetClassIndex) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json";

std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
std::shared_ptr<Dataset> ds =
Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
EXPECT_NE(ds, nullptr);

std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
@@ -355,7 +386,8 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";

std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, std::make_shared<SequentialSampler>(0, 6));
std::shared_ptr<Dataset> ds =
Coco(folder_path, annotation_file, "Stuff", false, std::make_shared<SequentialSampler>(0, 6));
EXPECT_NE(ds, nullptr);

// Create an iterator over the result of the above dataset
@@ -367,29 +399,33 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// std::string expect_file[] = {"000000391895", "000000318219", "000000554625",
// "000000574769", "000000060623", "000000309022"};
// std::vector<std::vector<float>> expect_segmentation_vector = {
// {10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
// 70.0, 72.0, 73.0, 74.0, 75.0, -1.0, -1.0, -1.0, -1.0, -1.0},
// {20.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0,
// 10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, -1.0},
// {40.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 40.0, 41.0, 42.0},
// {50.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0},
// {60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0},
// {60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}};
// std::vector<std::vector<dsize_t>> expect_size = {{2, 10}, {2, 11}, {1, 12}, {1, 13}, {1, 14}, {2, 7}};
std::string expect_file[] = {"000000391895", "000000318219", "000000554625",
"000000574769", "000000060623", "000000309022"};
std::vector<std::vector<float>> expect_segmentation_vector = {
{10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
70.0, 72.0, 73.0, 74.0, 75.0, -1.0, -1.0, -1.0, -1.0, -1.0},
{20.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0,
10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, -1.0},
{40.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 40.0, 41.0, 42.0},
{50.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0},
{60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0},
{60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}};
std::vector<std::vector<dsize_t>> expect_size = {{2, 10}, {2, 11}, {1, 12}, {1, 13}, {1, 14}, {2, 7}};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto segmentation = row["segmentation"];
// auto iscrowd = row["iscrowd"];
// mindspore::MSTensor expect_image;
// Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
// EXPECT_EQ(*image, *expect_image);
// mindspore::MSTensor expect_segmentation;
// Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &expect_segmentation);
// EXPECT_EQ(*segmentation, *expect_segmentation);
auto image = row["image"];
auto segmentation = row["segmentation"];

mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/" + expect_file[i] + ".jpg");
EXPECT_MSTENSOR_EQ(image, expect_image);

std::shared_ptr<Tensor> de_expect_segmentation;
ASSERT_OK(
Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &de_expect_segmentation));
mindspore::MSTensor expect_segmentation =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_segmentation));
EXPECT_MSTENSOR_EQ(segmentation, expect_segmentation);

iter->GetNextRow(&row);
i++;
}


+ 27
- 18
tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc View File

@@ -48,6 +48,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess1) {
i++;
// auto image = row["file_name"];
// MS_LOG(INFO) << "Tensor image file name: " << *image;

iter->GetNextRow(&row);
}

@@ -196,14 +197,16 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess5) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)0, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));

uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto label = row["label"];
auto label = row["label"];

// mindspore::MSTensor expected_item;
// Tensor::CreateScalar((int64_t)0, &expected_item);
// EXPECT_EQ(*expected_item, *label);
EXPECT_MSTENSOR_EQ(label, expect_item);

iter->GetNextRow(&row);
}
@@ -302,17 +305,19 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)999, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));

uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
// auto label = row["label"];
auto label = row["label"];
// MS_LOG(INFO) << "Tensor file name: " << *image;
// MS_LOG(INFO) << "Tensor label: " << *label;

// mindspore::MSTensor expected_item;
// Tensor::CreateScalar((int64_t)999, &expected_item);
// EXPECT_EQ(*expected_item, *label);
EXPECT_MSTENSOR_EQ(label, expect_item);

iter->GetNextRow(&row);
}
@@ -339,8 +344,8 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
EXPECT_NE(ds, nullptr);

std::vector<DataType> types = ToDETypes(ds->GetOutputTypes());
std::vector<TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes());
std::vector<mindspore::dataset::DataType> types = ToDETypes(ds->GetOutputTypes());
std::vector<mindspore::dataset::TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes());
std::vector<std::string> column_names = {"file_name", "label"};
EXPECT_EQ(types.size(), 2);
EXPECT_EQ(types[0].ToString(), "string");
@@ -371,17 +376,19 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)999, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));

uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
// auto label = row["label"];
auto label = row["label"];
// MS_LOG(INFO) << "Tensor file name: " << *image;
// MS_LOG(INFO) << "Tensor label: " << *label;

// mindspore::MSTensor expected_item;
// Tensor::CreateScalar((int64_t)999, &expected_item);
// EXPECT_EQ(*expected_item, *label);
EXPECT_MSTENSOR_EQ(label, expect_item);

iter->GetNextRow(&row);
}
@@ -444,15 +451,17 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)999, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));

uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto label = row["label"];
auto label = row["label"];
// MS_LOG(INFO) << "Tensor label: " << *label;

// mindspore::MSTensor expected_item;
// Tensor::CreateScalar((int64_t)999, &expected_item);
// EXPECT_EQ(*expected_item, *label);
EXPECT_MSTENSOR_EQ(label, expect_item);

iter->GetNextRow(&row);
}


+ 8
- 8
tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc View File

@@ -51,7 +51,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic1) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
@@ -109,7 +109,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasicWithPipeline) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
@@ -165,7 +165,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic2) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
// If no schema specified, RandomData will generate random columns
@@ -210,7 +210,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic3) {
std::vector<int64_t> expect_2d = {2, 2};
std::vector<int64_t> expect_3d = {2, 2, 2};

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
auto col_sint16 = row["col_sint16"];
@@ -292,7 +292,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic4) {
std::vector<int64_t> expect_2d = {2, 2};
std::vector<int64_t> expect_3d = {2, 2, 2};

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
auto col_sint16 = row["col_sint16"];
@@ -372,7 +372,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic5) {
std::vector<int64_t> expect_num = {1};
std::vector<int64_t> expect_1d = {2};

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
EXPECT_EQ(row.size(), 3);
@@ -427,7 +427,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic6) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
iter->GetNextRow(&row);
@@ -461,7 +461,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic7) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if RandomDataOp read correct columns
// Check if RandomData() read correct columns
uint64_t i = 0;
while (row.size() != 0) {
iter->GetNextRow(&row);


+ 27
- 25
tests/ut/cpp/dataset/c_api_dataset_voc_test.cc View File

@@ -15,6 +15,7 @@
*/
#include "common/common.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/core/tensor.h"

using namespace mindspore::dataset;
using mindspore::dataset::DataType;
@@ -47,20 +48,22 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if VOCOp read correct labels
// Check if VOC() read correct labels
// When we provide class_index, label of ["car","cat","train"] become [0,1,9]
// std::shared_ptr<Tensor> expect_label;
// Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label);
std::shared_ptr<Tensor> de_expect_label;
ASSERT_OK(Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &de_expect_label));
uint32_t expect[] = {9, 9, 9, 1, 1, 0};

// uint32_t expect[] = {9, 9, 9, 1, 1, 0};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
auto label = row["label"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
// expect_label->SetItemAt({0, 0}, expect[i]);
// EXPECT_EQ(*label, *expect_label);

ASSERT_OK(de_expect_label->SetItemAt({0, 0}, expect[i]));
mindspore::MSTensor expect_label = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_label));
EXPECT_MSTENSOR_EQ(label, expect_label);

iter->GetNextRow(&row);
i++;
@@ -132,9 +135,13 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if VOCOp read correct images/labels
// std::string expect_file[] = {"15", "32", "33", "39"};
// uint32_t expect_num[] = {5, 5, 4, 3};
// Check if VOC() read correct images/labels
std::string expect_file[] = {"15", "32", "33", "39"};
uint32_t expect_num[] = {5, 5, 4, 3};

std::shared_ptr<Tensor> de_expect_label;
ASSERT_OK(Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &de_expect_label));

uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
@@ -142,14 +149,12 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor label shape: " << label.Shape();

// std::shared_ptr<Tensor> expect_image;
// Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image);
// EXPECT_EQ(*image, *expect_image);
mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg");
EXPECT_MSTENSOR_EQ(image, expect_image);

// std::shared_ptr<Tensor> expect_label;
// Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label);
// expect_label->SetItemAt({0, 0}, expect_num[i]);
// EXPECT_EQ(*label, *expect_label);
ASSERT_OK(de_expect_label->SetItemAt({0, 0}, expect_num[i]));
mindspore::MSTensor expect_label = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_label));
EXPECT_MSTENSOR_EQ(label, expect_label);

iter->GetNextRow(&row);
i++;
@@ -205,9 +210,8 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if VOCOp read correct images/targets
// using Tensor = mindspore::dataset::Tensor;
// std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
// Check if VOC() read correct images/targets
std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
@@ -215,13 +219,11 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor target shape: " << target.Shape();

// std::shared_ptr<Tensor> expect_image;
// Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image);
// EXPECT_EQ(*image, *expect_image);
mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg");
EXPECT_MSTENSOR_EQ(image, expect_image);

// std::shared_ptr<Tensor> expect_target;
// Tensor::CreateFromFile(folder_path + "/SegmentationClass/" + expect_file[i] + ".png", &expect_target);
// EXPECT_EQ(*target, *expect_target);
mindspore::MSTensor expect_target = ReadFileToTensor(folder_path + "/SegmentationClass/" + expect_file[i] + ".png");
EXPECT_MSTENSOR_EQ(target, expect_target);

iter->GetNextRow(&row);
i++;


+ 18
- 16
tests/ut/cpp/dataset/c_api_datasets_test.cc View File

@@ -15,6 +15,7 @@
*/
#include "common/common.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/core/tensor.h"

using namespace mindspore::dataset;
using mindspore::dataset::Tensor;
@@ -43,25 +44,26 @@ TEST_F(MindDataTestPipeline, TestCelebADataset) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if CelebAOp read correct images/attr
// std::string expect_file[] = {"1.JPEG", "2.jpg"};
// std::vector<std::vector<uint32_t>> expect_attr_vector = {
// {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
// 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
// {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
// 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}};
// Check if CelebA() read correct images/attr
std::string expect_file[] = {"1.JPEG", "2.jpg"};
std::vector<std::vector<uint32_t>> expect_attr_vector = {
{0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
{0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto attr = row["attr"];
auto image = row["image"];
auto attr = row["attr"];

// std::shared_ptr<Tensor> expect_image;
// Tensor::CreateFromFile(folder_path + expect_file[i], &expect_image);
// EXPECT_EQ(*image, *expect_image);
mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + expect_file[i]);
EXPECT_MSTENSOR_EQ(image, expect_image);

// std::shared_ptr<Tensor> expect_attr;
// Tensor::CreateFromVector(expect_attr_vector[i], TensorShape({40}), &expect_attr);
// EXPECT_EQ(*attr, *expect_attr);
std::shared_ptr<Tensor> de_expect_attr;
ASSERT_OK(Tensor::CreateFromVector(expect_attr_vector[i], TensorShape({40}), &de_expect_attr));
mindspore::MSTensor expect_attr =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_attr));
EXPECT_MSTENSOR_EQ(attr, expect_attr);

iter->GetNextRow(&row);
i++;
@@ -90,7 +92,7 @@ TEST_F(MindDataTestPipeline, TestCelebADefault) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);

// Check if CelebAOp read correct images/attr
// Check if CelebA() read correct images/attr
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];


+ 2
- 2
tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc View File

@@ -142,8 +142,8 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess1) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// std::shared_ptr<TensorTransform> image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}



+ 32
- 0
tests/ut/cpp/dataset/common/common.cc View File

@@ -14,6 +14,10 @@
* limitations under the License.
*/
#include "common.h"
#include <fstream>
#include <algorithm>
#include <string>
#include <vector>

namespace UT {
#ifdef __cplusplus
@@ -44,6 +48,34 @@ std::vector<mindspore::dataset::DataType> DatasetOpTesting::ToDETypes(const std:
return ret_t;
}

// Function to read a file into an MSTensor
// Note: This provides the analogous support for DETensor's CreateFromFile.
mindspore::MSTensor DatasetOpTesting::ReadFileToTensor(const std::string &file) {
if (file.empty()) {
MS_LOG(ERROR) << "Pointer file is nullptr; return an empty Tensor.";
return mindspore::MSTensor();
}
std::ifstream ifs(file);
if (!ifs.good()) {
MS_LOG(ERROR) << "File: " << file << " does not exist; return an empty Tensor.";
return mindspore::MSTensor();
}
if (!ifs.is_open()) {
MS_LOG(ERROR) << "File: " << file << " open failed; return an empty Tensor.";
return mindspore::MSTensor();
}

ifs.seekg(0, std::ios::end);
size_t size = ifs.tellg();
mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);

ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
ifs.close();

return buf;
}

#ifdef __cplusplus
#if __cplusplus
}


+ 13
- 3
tests/ut/cpp/dataset/common/common.h View File

@@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
#ifndef TESTS_UT_CPP_DATASET_COMMON_COMMON_H_
#define TESTS_UT_CPP_DATASET_COMMON_COMMON_H_

#include "gtest/gtest.h"
#include "include/api/status.h"
@@ -62,6 +62,15 @@ using mindspore::StatusCode;
} \
} while (false)

// Macro to compare 2 MSTensors; compare shape-size and data
#define EXPECT_MSTENSOR_EQ(_mstensor1, _mstensor2) \
do { \
EXPECT_EQ(_mstensor1.Shape().size(), _mstensor2.Shape().size()); \
EXPECT_EQ(_mstensor1.DataSize(), _mstensor2.DataSize()); \
EXPECT_EQ(std::memcmp((const void *)_mstensor1.Data().get(), (const void *)_mstensor2.Data().get(), \
_mstensor2.DataSize()), 0); \
} while (false)

namespace UT {
class Common : public testing::Test {
public:
@@ -75,9 +84,10 @@ class DatasetOpTesting : public Common {
public:
std::vector<mindspore::dataset::TensorShape> ToTensorShapeVec(const std::vector<std::vector<int64_t>> &v);
std::vector<mindspore::dataset::DataType> ToDETypes(const std::vector<mindspore::DataType> &t);
mindspore::MSTensor ReadFileToTensor(const std::string &file);
std::string datasets_root_path_;
std::string mindrecord_root_path_;
void SetUp() override;
};
} // namespace UT
#endif // TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
#endif // TESTS_UT_CPP_DATASET_COMMON_COMMON_H_

+ 1
- 32
tests/ut/cpp/dataset/execute_test.cc View File

@@ -13,9 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include "common/common.h"
#include "common/cvop_common.h"
#include "include/api/types.h"
#include "minddata/dataset/core/de_tensor.h"
#include "minddata/dataset/include/execute.h"
@@ -28,39 +26,10 @@ using mindspore::LogStream;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::MsLogLevel::INFO;

class MindDataTestExecute : public UT::CVOP::CVOpCommon {
class MindDataTestExecute : public UT::DatasetOpTesting {
protected:
MindDataTestExecute() : CVOpCommon() {}

std::shared_ptr<Tensor> output_tensor_;
};

mindspore::MSTensor ReadFileToTensor(const std::string &file) {
if (file.empty()) {
MS_LOG(ERROR) << "Pointer file is nullptr, return an empty Tensor.";
return mindspore::MSTensor();
}
std::ifstream ifs(file);
if (!ifs.good()) {
MS_LOG(ERROR) << "File: " << file << " does not exist, return an empty Tensor.";
return mindspore::MSTensor();
}
if (!ifs.is_open()) {
MS_LOG(ERROR) << "File: " << file << "open failed, return an empty Tensor.";
return mindspore::MSTensor();
}

ifs.seekg(0, std::ios::end);
size_t size = ifs.tellg();
mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);

ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
ifs.close();

return buf;
}

TEST_F(MindDataTestExecute, TestComposeTransforms) {
MS_LOG(INFO) << "Doing TestComposeTransforms.";



Write Preview
Loading…
Cancel
Save