From 97cf4e516eeadf15f4f34732c1eb6d6ed660c2e8 Mon Sep 17 00:00:00 2001 From: hesham Date: Tue, 6 Apr 2021 14:57:26 -0400 Subject: [PATCH] Address 3 code issues: 1- access out of bound memory in toFlatIndex 2- CreateFromNPArray function is 50> 3- duplicate variable init --- .../ccsrc/minddata/dataset/core/tensor.cc | 96 ++++++++----------- .../ccsrc/minddata/dataset/core/tensor_row.h | 2 +- .../minddata/dataset/core/tensor_shape.cc | 5 + .../dataset/engine/consumers/tree_consumer.cc | 2 +- .../engine/datasetops/source/generator_op.cc | 2 +- .../text/kernels/wordpiece_tokenizer_op.cc | 2 +- tests/ut/cpp/dataset/album_op_test.cc | 14 +-- .../ut/cpp/dataset/c_api_dataset_clue_test.cc | 22 ++--- .../cpp/dataset/c_api_dataset_config_test.cc | 4 +- .../ut/cpp/dataset/c_api_dataset_csv_test.cc | 22 ++--- .../ut/cpp/dataset/c_api_dataset_ops_test.cc | 24 ++--- .../dataset/c_api_dataset_textfile_test.cc | 20 ++-- tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc | 8 +- tests/ut/cpp/dataset/pad_end_op_test.cc | 2 +- 14 files changed, 106 insertions(+), 119 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.cc b/mindspore/ccsrc/minddata/dataset/core/tensor.cc index 218cca9c6d..1d77fc62be 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.cc @@ -59,7 +59,28 @@ namespace dataset { out << std::hex << std::setw(2) << std::setfill('0') << o << std::dec << std::setfill(' '); \ break; \ } - +/// Copy memory with no max limit since memcpy_s will fail when byte_size > 2^31 - 1 (SECUREC_MEM_MAX_LEN). +/// \param dest Destination buffer. +/// \param destMax Size of the destination buffer. +/// \param src Buffer to copy from. +/// \param count Number of characters to copy +/// \return Error number. Returns 0 for succuss copying. +errno_t memcpy_ss(uchar *dest, size_t destMax, const uchar *src, size_t count) { + uint32_t step = 0; + while (count >= SECUREC_MEM_MAX_LEN) { + int ret_code = memcpy_s(dest + step * SECUREC_MEM_MAX_LEN, SECUREC_MEM_MAX_LEN, src + step * SECUREC_MEM_MAX_LEN, + SECUREC_MEM_MAX_LEN); + if (ret_code != 0) { + return ret_code; + } + count -= SECUREC_MEM_MAX_LEN; + step++; + } + if (count > 0) { + return memcpy_s(dest + step * SECUREC_MEM_MAX_LEN, count, src + step * SECUREC_MEM_MAX_LEN, count); + } + return 0; +} Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) { // grab the mem pool from global context and create the allocator for char data area std::shared_ptr global_pool = GlobalContext::Instance()->mem_pool(); @@ -111,7 +132,7 @@ Status Tensor::CreateFromMemory(const TensorShape &shape, const DataType &type, if (src != nullptr) { // Given the shape/type of this tensor, compute the data size and copy in the input bytes. int64_t byte_size = (*out)->SizeInBytes(); - int ret_code = memcpy_s((*out)->data_, byte_size, src, byte_size); + int ret_code = memcpy_ss((*out)->data_, byte_size, src, byte_size); CHECK_FAIL_RETURN_UNEXPECTED(ret_code == 0, "Failed to copy data into tensor."); } return Status::OK(); @@ -163,71 +184,32 @@ Status Tensor::CreateFromNpArray(const py::array &arr, std::shared_ptr * if (DataType::FromNpArray(arr) == DataType::DE_STRING) { return CreateFromNpString(arr, out); } - const TensorAlloc *alloc = GlobalContext::Instance()->tensor_allocator(); - *out = std::allocate_shared(*alloc, TensorShape::CreateScalar(), DataType(DataType::DE_UNKNOWN)); std::vector shape; - for (dsize_t i = 0; i < arr.ndim(); i++) { - shape.push_back(static_cast(arr.shape()[i])); - } - - (*out)->shape_ = TensorShape(shape); - (*out)->type_ = DataType::FromNpArray(arr); - if (!(*out)->shape_.known()) RETURN_STATUS_UNEXPECTED("Invalid shape."); - - if ((*out)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type."); - - std::shared_ptr global_pool = GlobalContext::Instance()->mem_pool(); - (*out)->data_allocator_ = std::make_unique>(global_pool); - int64_t byte_size = (*out)->SizeInBytes(); - if (byte_size == 0) { - return Status::OK(); - } - - RETURN_IF_NOT_OK((*out)->AllocateBuffer(byte_size)); - - unsigned char *data = static_cast(arr.request().ptr); - if ((*out)->data_ == nullptr) { - RETURN_STATUS_UNEXPECTED("Failed to create memory for Tensor."); - } - std::vector strides; - for (dsize_t i = 0; i < arr.ndim(); i++) { - strides.push_back(static_cast(arr.strides()[i])); - } - // check if strides are contiguous bool is_strided = false; - dsize_t count = (*out)->shape_.NumOfElements(); - for (size_t i = 0; i < shape.size(); i++) { - count /= shape[i]; - if (strides[i] != (*out)->type_.SizeInBytes() * count) { - is_strided = true; - break; + dsize_t count = arr.size(); + for (dsize_t i = 0; i < arr.ndim(); i++) { + shape.push_back(static_cast(arr.shape()[i])); + strides.push_back(static_cast(arr.strides()[i])); + // in case of empty array num_items=0 + if (count != 0) { + count /= shape[i]; + if (strides[i] != arr.itemsize() * count) { + is_strided = true; + } } } + unsigned char *data = static_cast(arr.request().ptr); + if (is_strided) { + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape(shape), DataType::FromNpArray(arr), out)); RETURN_IF_NOT_OK(CopyStridedArray((*out)->data_, data, shape, strides, (*out)->type_.SizeInBytes())); } else { - // fix: memcpy_s will fail when byte_size > 2^31 - 1 - uint32_t step = 0; - while (byte_size >= kDeMaxDim) { - int ret_code = memcpy_s((*out)->data_ + step * kDeMaxDim, kDeMaxDim, data + step * kDeMaxDim, kDeMaxDim); - if (ret_code != 0) { - RETURN_STATUS_UNEXPECTED("Failed to copy data into Tensor."); - } - byte_size -= kDeMaxDim; - step++; - } - if (byte_size > 0) { - int ret_code = memcpy_s((*out)->data_ + step * kDeMaxDim, byte_size, data + step * kDeMaxDim, byte_size); - if (ret_code != 0) { - RETURN_STATUS_UNEXPECTED("Failed to copy data into Tensor."); - } - } + RETURN_IF_NOT_OK(Tensor::CreateFromMemory(TensorShape(shape), DataType::FromNpArray(arr), data, out)); } - return Status::OK(); } #endif @@ -943,8 +925,8 @@ Status Tensor::Slice(std::shared_ptr *out, const std::vectorshape() != TensorShape({})) { RETURN_STATUS_UNEXPECTED("ConvertFromTensorRow: The input tensors must be a scalar tensor."); } - return input.at(0)->GetItemAt(o, {0}); + return input.at(0)->GetItemAt(o, {}); } /// Convert a TensorRow consisting of one 1-D tensor to a vector of size n. diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc index 67e43d4f59..3286b19671 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc @@ -230,6 +230,11 @@ std::vector TensorShape::Strides() const { return std::vector{ // Name: ToFlatIndex() // Description: convert a vector style index to number, used to access memory internal use only Status TensorShape::ToFlatIndex(const std::vector &index, dsize_t *flat_index) const { + if (index.size() != raw_shape_.size()) { + std::stringstream ss; + ss << "Index size (" << index.size() << ") does not match the shape size (" << raw_shape_.size() << ")."; + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, ss.str()); + } *flat_index = 0; for (size_t k = 0; k < index.size(); k++) { *flat_index += diff --git a/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc b/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc index b7920a3ebb..d13ae192bf 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc @@ -405,7 +405,7 @@ Status SaveToDisk::FetchItemData(std::shared_ptr tensor, std::string col RETURN_IF_NOT_OK(s); } else if (column_type == DataType::DE_STRING) { std::string_view sv; - RETURN_IF_NOT_OK(tensor->GetItemAt(&sv, {0})); // assume scalar string tensor + RETURN_IF_NOT_OK(tensor->GetItemAt(&sv, {})); // assume scalar string tensor std::string ss(sv); (*row_raw_data)[column_name] = std::move(ss); return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc index ff0b5de6e9..b2e50b840a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc @@ -81,7 +81,7 @@ Status GeneratorOp::InitSampler() { // Invoke the generatorFunction to get generator object Status GeneratorOp::CreateGeneratorObject() { - Status ret = Status::OK(); + Status ret; { // Acquire Python GIL py::gil_scoped_acquire gil_acquire; diff --git a/mindspore/ccsrc/minddata/dataset/text/kernels/wordpiece_tokenizer_op.cc b/mindspore/ccsrc/minddata/dataset/text/kernels/wordpiece_tokenizer_op.cc index 6d919112d4..b8226ca24d 100644 --- a/mindspore/ccsrc/minddata/dataset/text/kernels/wordpiece_tokenizer_op.cc +++ b/mindspore/ccsrc/minddata/dataset/text/kernels/wordpiece_tokenizer_op.cc @@ -128,7 +128,7 @@ Status WordpieceTokenizerOp::Compute(const TensorRow &input, TensorRow *output) uint32_t basic_start = 0; std::vector temp_tokens; if (with_offsets_ && input.size() == 3) { - RETURN_IF_NOT_OK(input[1]->GetItemAt(&basic_start, {count, 0})); + RETURN_IF_NOT_OK(input[1]->GetItemAt(&basic_start, {count})); } RETURN_IF_NOT_OK(GetTokens(std::string(*iter), basic_start, &temp_tokens, &offsets_start, &offsets_limit)); out_tokens.insert(out_tokens.end(), temp_tokens.begin(), temp_tokens.end()); diff --git a/tests/ut/cpp/dataset/album_op_test.cc b/tests/ut/cpp/dataset/album_op_test.cc index b28d3ee2e0..4c081707f7 100644 --- a/tests/ut/cpp/dataset/album_op_test.cc +++ b/tests/ut/cpp/dataset/album_op_test.cc @@ -92,7 +92,7 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchema) { uint64_t i = 0; std::string_view label = 0; while (tensor_map.size() != 0) { - EXPECT_TRUE(tensor_map["label"]->GetItemAt(&label, {})); + EXPECT_TRUE(tensor_map["label"]->GetItemAt(&label, {0})); MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" << tensor_map["label"] << "\n"; i++; @@ -118,7 +118,7 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaNoOrder) { uint64_t i = 0; std::string_view label; while (tensor_map.size() != 0) { - EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {})); + EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {0})); MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" << tensor_map["label"] << "\n"; i++; @@ -146,10 +146,10 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithSchemaFloat) { std::string_view label; double priority = 0; while (tensor_map.size() != 0) { - EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {})); - EXPECT_OK(tensor_map["_priority"]->GetItemAt(&priority, {})); + EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {0})); + EXPECT_OK(tensor_map["_priority"]->GetItemAt(&priority, {0})); MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" - << tensor_map["label"] << "priority: " << priority << "\n"; + << tensor_map["label"] << "priority: " << priority << "\n"; i++; di.GetNextAsMap(&tensor_map); } @@ -176,8 +176,8 @@ TEST_F(MindDataTestAlbum, TestSequentialAlbumWithFullSchema) { double priority = 0; int64_t id = 0; while (tensor_map.size() != 0) { - EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {})); - EXPECT_OK(tensor_map["_priority"]->GetItemAt(&priority, {})); + EXPECT_OK(tensor_map["label"]->GetItemAt(&label, {0})); + EXPECT_OK(tensor_map["_priority"]->GetItemAt(&priority, {0})); EXPECT_OK(tensor_map["id"]->GetItemAt(&id, {})); MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "label shape" << tensor_map["label"] << "priority: " << priority diff --git a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc index 1186f6ff85..ec3ddc864d 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc @@ -57,7 +57,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); @@ -86,7 +86,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); iter->GetNextRow(&row); @@ -109,7 +109,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); iter->GetNextRow(&row); @@ -252,7 +252,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCMNLI) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); @@ -295,7 +295,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCSL) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); @@ -438,7 +438,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); @@ -500,7 +500,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFilesA) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -567,7 +567,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFilesB) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); // Compare against expected result EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); @@ -622,7 +622,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); @@ -669,7 +669,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); @@ -713,7 +713,7 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); diff --git a/tests/ut/cpp/dataset/c_api_dataset_config_test.cc b/tests/ut/cpp/dataset/c_api_dataset_config_test.cc index a71a9547e2..12f707a8ae 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_config_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_config_test.cc @@ -141,7 +141,7 @@ TEST_F(MindDataTestPipeline, TestShuffleWithSeed) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -206,7 +206,7 @@ TEST_F(MindDataTestPipeline, TestCallShuffleTwice) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); diff --git a/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc b/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc index 9b074ab9ee..331346ce77 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc @@ -57,7 +57,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetBasic) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } @@ -123,7 +123,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetMultiFiles) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } @@ -169,7 +169,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetNumSamples) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } @@ -211,7 +211,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetDistribution) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } @@ -273,15 +273,15 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetType) { ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); if (colum_type[j]->type == CsvType::INT) { int val; - de_text->GetItemAt(&val, {0}); + de_text->GetItemAt(&val, {}); EXPECT_EQ(val, std::dynamic_pointer_cast>(expected[i][j])->value); } else if (colum_type[j]->type == CsvType::FLOAT) { float val; - de_text->GetItemAt(&val, {0}); + de_text->GetItemAt(&val, {}); EXPECT_EQ(val, std::dynamic_pointer_cast>(expected[i][j])->value); } else if (colum_type[j]->type == CsvType::STRING) { std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), std::dynamic_pointer_cast>(expected[i][j])->value.c_str()); } @@ -326,7 +326,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetHeader) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } @@ -435,7 +435,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetShuffleFilesA) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } @@ -492,7 +492,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetShuffleFilesB) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); @@ -548,7 +548,7 @@ TEST_F(MindDataTestPipeline, TestCSVDatasetShuffleGlobal) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); EXPECT_STREQ(ss.c_str(), expected_result[i][j].c_str()); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index 07ac14e272..3eedcc412c 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -66,9 +66,9 @@ MSTensorVec Predicate1(MSTensorVec in) { // Convert from boolean to TensorRow TensorRow output; std::shared_ptr out; - Tensor::CreateEmpty(mindspore::dataset::TensorShape({1}), + Tensor::CreateEmpty(mindspore::dataset::TensorShape({}), mindspore::dataset::DataType(mindspore::dataset::DataType::Type::DE_BOOL), &out); - out->SetItemAt({0}, result); + out->SetItemAt({}, result); output.push_back(out); return RowToVec(output); @@ -85,9 +85,9 @@ MSTensorVec Predicate2(MSTensorVec in) { // Convert from boolean to TensorRow TensorRow output; std::shared_ptr out; - Tensor::CreateEmpty(mindspore::dataset::TensorShape({1}), + Tensor::CreateEmpty(mindspore::dataset::TensorShape({}), mindspore::dataset::DataType(mindspore::dataset::DataType::Type::DE_BOOL), &out); - out->SetItemAt({0}, result); + out->SetItemAt({}, result); output.push_back(out); return RowToVec(output); @@ -191,14 +191,14 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) { // Iterate over the dataset and get each row std::unordered_map row; - iter->GetNextRow(&row); + ASSERT_OK(iter->GetNextRow(&row)); uint64_t i = 0; while (row.size() != 0) { i++; auto image = row["image"]; MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); - iter->GetNextRow(&row); + ASSERT_OK(iter->GetNextRow(&row)); } // With 2 boundaries, 3 buckets are created EXPECT_EQ(i, 3); @@ -630,7 +630,7 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess1) { // iterate over the dataset and get each row std::unordered_map row; - iter->GetNextRow(&row); + ASSERT_OK(iter->GetNextRow(&row)); std::vector label_list; uint64_t i = 0; @@ -641,10 +641,10 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess1) { std::shared_ptr de_label; uint64_t label_value; ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label)); - de_label->GetItemAt(&label_value, {0}); + ASSERT_OK(de_label->GetItemAt(&label_value, {0})); label_list.push_back(label_value); - iter->GetNextRow(&row); + ASSERT_OK(iter->GetNextRow(&row)); } // Only 1 column whose label is equal to 3 @@ -676,7 +676,7 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess2) { // iterate over the dataset and get each row std::unordered_map row; - iter->GetNextRow(&row); + ASSERT_OK(iter->GetNextRow(&row)); std::vector label_list; uint64_t i = 0; @@ -687,10 +687,10 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess2) { std::shared_ptr de_label; uint64_t label_value; ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label)); - de_label->GetItemAt(&label_value, {0}); + ASSERT_OK(de_label->GetItemAt(&label_value, {0})); label_list.push_back(label_value); - iter->GetNextRow(&row); + ASSERT_OK(iter->GetNextRow(&row)); } // There are 2 columns whose label is more than 1 diff --git a/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc b/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc index cf0fe29034..fceca04005 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc @@ -63,7 +63,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -323,7 +323,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1A) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -384,7 +384,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1B) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -444,7 +444,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -507,7 +507,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1A) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -570,7 +570,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1B) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -632,7 +632,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -690,7 +690,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -751,7 +751,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result @@ -812,7 +812,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { std::shared_ptr de_text; ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); std::string_view sv; - de_text->GetItemAt(&sv, {0}); + de_text->GetItemAt(&sv, {}); std::string ss(sv); MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); // Compare against expected result diff --git a/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc b/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc index f012966233..98a8b09edc 100644 --- a/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc +++ b/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc @@ -54,7 +54,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) { std::shared_ptr de_label; int64_t label_value; ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label)); - de_label->GetItemAt(&label_value, {0}); + de_label->GetItemAt(&label_value, {}); EXPECT_TRUE(img_class[(i % sampler_size) / class_size] == label_value); iter->GetNextRow(&row); @@ -100,7 +100,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) { std::shared_ptr de_label; int64_t label_value; ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label)); - de_label->GetItemAt(&label_value, {0}); + de_label->GetItemAt(&label_value, {}); EXPECT_TRUE(label_value >= 0 && label_value <= 3); iter->GetNextRow(&row); @@ -149,7 +149,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) { std::shared_ptr de_label; int64_t label_value; ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label)); - de_label->GetItemAt(&label_value, {0}); + de_label->GetItemAt(&label_value, {}); EXPECT_TRUE(label_value >= 0 && label_value <= 3); iter->GetNextRow(&row); @@ -198,7 +198,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) { std::shared_ptr de_label; int64_t label_value; ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label)); - de_label->GetItemAt(&label_value, {0}); + de_label->GetItemAt(&label_value, {}); EXPECT_TRUE(label_value >= 0 && label_value <= 3); iter->GetNextRow(&row); diff --git a/tests/ut/cpp/dataset/pad_end_op_test.cc b/tests/ut/cpp/dataset/pad_end_op_test.cc index b4bd993f38..915a8dda36 100644 --- a/tests/ut/cpp/dataset/pad_end_op_test.cc +++ b/tests/ut/cpp/dataset/pad_end_op_test.cc @@ -32,7 +32,7 @@ TEST_F(MindDataTestPadEndOp, TestOp) { // first set of testunits for numeric values - TensorShape pad_data_shape({1}); + TensorShape pad_data_shape({}); // prepare input tensor std::vector orig1 = {1, 1, 1, 1};