|
|
|
@@ -90,56 +90,44 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchema) { |
|
|
|
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; |
|
|
|
std::vector<std::string> column_names = {"image", "label", "id"}; |
|
|
|
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file, column_names, false), Repeat(2)}); |
|
|
|
tree->Prepare(); |
|
|
|
Status rc = tree->Launch(); |
|
|
|
if (rc.IsError()) { |
|
|
|
MS_LOG(ERROR) << "Return code error detected during tree launch: " << "."; |
|
|
|
EXPECT_TRUE(false); |
|
|
|
} else { |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(tree->Prepare()); |
|
|
|
ASSERT_OK(tree->Launch()); |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(di.GetNextAsMap(&tensor_map)); |
|
|
|
uint64_t i = 0; |
|
|
|
std::string_view label = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
EXPECT_TRUE(tensor_map["label"]->GetItemAt(&label, {})); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
EXPECT_TRUE(rc.IsOk()); |
|
|
|
uint64_t i = 0; |
|
|
|
int32_t label = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows" << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows: " << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
|
|
|
|
TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaNoOrder) { |
|
|
|
std::string folder_path = datasets_root_path_ + "/testAlbum/images"; |
|
|
|
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; |
|
|
|
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)}); |
|
|
|
tree->Prepare(); |
|
|
|
Status rc = tree->Launch(); |
|
|
|
if (rc.IsError()) { |
|
|
|
MS_LOG(ERROR) << "Return code error detected during tree launch: " << "."; |
|
|
|
EXPECT_TRUE(false); |
|
|
|
} else { |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(tree->Prepare()); |
|
|
|
ASSERT_OK(tree->Launch()); |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_TRUE(di.GetNextAsMap(&tensor_map)); |
|
|
|
uint64_t i = 0; |
|
|
|
std::string_view label; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {})); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
EXPECT_TRUE(rc.IsOk()); |
|
|
|
uint64_t i = 0; |
|
|
|
int32_t label = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows" << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows: " << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
|
|
|
|
TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) { |
|
|
|
@@ -148,29 +136,23 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) { |
|
|
|
std::string schema_file = datasets_root_path_ + "/testAlbum/floatSchema.json"; |
|
|
|
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)}); |
|
|
|
tree->Prepare(); |
|
|
|
Status rc = tree->Launch(); |
|
|
|
if (rc.IsError()) { |
|
|
|
MS_LOG(ERROR) << "Return code error detected during tree launch: " << "."; |
|
|
|
EXPECT_TRUE(false); |
|
|
|
} else { |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(tree->Launch()); |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(di.GetNextAsMap(&tensor_map)); |
|
|
|
uint64_t i = 0; |
|
|
|
std::string_view label; |
|
|
|
double priority = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {})); |
|
|
|
EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {})); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "priority: " << priority << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
EXPECT_TRUE(rc.IsOk()); |
|
|
|
uint64_t i = 0; |
|
|
|
int32_t label = 0; |
|
|
|
double priority = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); |
|
|
|
tensor_map["_priority"]->GetItemAt<double>(&priority, {}); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "priority: " << priority << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows" << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows: " << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
|
|
|
|
TEST_F(MindDataTestAlbum, TestSequentialAlbumWithFullSchema) { |
|
|
|
@@ -178,32 +160,26 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithFullSchema) { |
|
|
|
// add the priority column |
|
|
|
std::string schema_file = datasets_root_path_ + "/testAlbum/fullSchema.json"; |
|
|
|
auto tree = Build({AlbumSchema(16, 2, 32, folder_path, schema_file), Repeat(2)}); |
|
|
|
tree->Prepare(); |
|
|
|
Status rc = tree->Launch(); |
|
|
|
if (rc.IsError()) { |
|
|
|
MS_LOG(ERROR) << "Return code error detected during tree launch: " << "."; |
|
|
|
EXPECT_TRUE(false); |
|
|
|
} else { |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(tree->Prepare()); |
|
|
|
ASSERT_OK(tree->Launch()); |
|
|
|
DatasetIterator di(tree); |
|
|
|
TensorMap tensor_map; |
|
|
|
ASSERT_OK(di.GetNextAsMap(&tensor_map)); |
|
|
|
uint64_t i = 0; |
|
|
|
std::string_view label = 0; |
|
|
|
double priority = 0; |
|
|
|
int64_t id = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {})); |
|
|
|
EXPECT_OK(tensor_map["_priority"]->GetItemAt<double>(&priority, {})); |
|
|
|
EXPECT_OK(tensor_map["id"]->GetItemAt<int64_t>(&id, {})); |
|
|
|
MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "priority: " << priority << " embedding : " |
|
|
|
<< tensor_map["_embedding"]->shape() << " id: " << id << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
EXPECT_TRUE(rc.IsOk()); |
|
|
|
uint64_t i = 0; |
|
|
|
int32_t label = 0; |
|
|
|
double priority = 0; |
|
|
|
int64_t id = 0; |
|
|
|
while (tensor_map.size() != 0) { |
|
|
|
tensor_map["label"]->GetItemAt<int32_t>(&label, {}); |
|
|
|
tensor_map["_priority"]->GetItemAt<double>(&priority, {}); |
|
|
|
tensor_map["id"]->GetItemAt<int64_t>(&id, {}); |
|
|
|
MS_LOG(ERROR) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" |
|
|
|
<< tensor_map["label"] << "priority: " << priority << " embedding : " << |
|
|
|
tensor_map["_embedding"]->shape() << " id: " << id << "\n"; |
|
|
|
i++; |
|
|
|
di.GetNextAsMap(&tensor_map); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows" << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
MS_LOG(INFO) << "got rows: " << i << "\n"; |
|
|
|
EXPECT_TRUE(i == 14); |
|
|
|
} |
|
|
|
|