Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: db80643ee2
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'db80643ee2'
${ noResults }
mindspore/tests/ut/cpp/dataset/batch_op_test.cc

373 lines
14 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2019 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. #include <iostream>

  17. #include <memory>

  18. #include <string>

  19. #include "minddata/dataset/core/client.h"

  20. #include "common/common.h"

  21. #include "common/utils.h"

  22. #include "gtest/gtest.h"

  23. #include "minddata/dataset/core/global_context.h"

  24. #include "utils/log_adapter.h"

  25. #include "securec.h"

  26. #include "minddata/dataset/util/status.h"

  27. 

  28. namespace common = mindspore::common;

  29. namespace de = mindspore::dataset;

  30. 

  31. using namespace mindspore::dataset;

  32. using mindspore::LogStream;

  33. using mindspore::ExceptionType::NoExceptionType;

  34. using mindspore::MsLogLevel::ERROR;

  35. 

  36. class MindDataTestBatchOp : public UT::DatasetOpTesting {

  37.  protected:

  38. };

  39. 

  40. std::shared_ptr<de::BatchOp> Batch(int32_t batch_size = 1, bool drop = false, int rows_per_buf = 2) {

  41.   Status rc;

  42.   std::shared_ptr<de::BatchOp> op;

  43.   rc = de::BatchOp::Builder(batch_size).SetDrop(drop).Build(&op);

  44.   EXPECT_TRUE(rc.IsOk());

  45.   return op;

  46. }

  47. 

  48. std::shared_ptr<de::RepeatOp> Repeat(int repeat_cnt = 1) {

  49.   de::RepeatOp::Builder builder(repeat_cnt);

  50.   std::shared_ptr<de::RepeatOp> op;

  51.   Status rc = builder.Build(&op);

  52.   EXPECT_TRUE(rc.IsOk());

  53.   return op;

  54. }

  55. 

  56. std::shared_ptr<de::TFReaderOp> TFReader(std::string schema, int rows_per_buf = 2, int num_works = 8) {

  57.   std::shared_ptr<de::TFReaderOp> so;

  58.   de::TFReaderOp::Builder builder;

  59.   builder.SetDatasetFilesList({schema}).SetRowsPerBuffer(rows_per_buf).SetNumWorkers(num_works);

  60.   Status rc = builder.Build(&so);

  61.   return so;

  62. }

  63. 

  64. std::shared_ptr<de::ExecutionTree> Build(std::vector<std::shared_ptr<de::DatasetOp>> ops) {

  65.   std::shared_ptr<de::ExecutionTree> tree = std::make_shared<de::ExecutionTree>();

  66.   for (int i = 0; i < ops.size(); i++) {

  67.     tree->AssociateNode(ops[i]);

  68.     if (i > 0) {

  69.       ops[i]->AddChild(ops[i - 1]);

  70.     }

  71.     if (i == ops.size() - 1) {

  72.       tree->AssignRoot(ops[i]);

  73.     }

  74.   }

  75.   return tree;

  76. }

  77. 

  78. TEST_F(MindDataTestBatchOp, TestSimpleBatch) {

  79.   std::string schema_file = datasets_root_path_ + "/testBatchDataset/test.data";

  80.   bool success = false;

  81.   auto tree = Build({TFReader(schema_file), Batch(12)});

  82.   tree->Prepare();

  83.   Status rc = tree->Launch();

  84.   if (rc.IsError()) {

  85.     MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";

  86.   } else {

  87.     int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807};

  88.     de::DatasetIterator di(tree);

  89.     TensorMap tensor_map;

  90.     rc = di.GetNextAsMap(&tensor_map);

  91.     EXPECT_TRUE(rc.IsOk());

  92.     std::shared_ptr<de::Tensor> t;

  93.     rc = de::Tensor::CreateTensor(&t, TensorImpl::kFlexible, de::TensorShape({12, 1}), de::DataType(DataType::DE_INT64),

  94.                                   (unsigned char *)payload);

  95.     EXPECT_TRUE(rc.IsOk());

  96.     // verify the actual data in Tensor is correct

  97.     EXPECT_EQ(*t == *tensor_map["col_sint64"], true);

  98.     // change what's in Tensor and verify this time the data is incorrect1;

  99.     EXPECT_EQ(*t == *tensor_map["col_sint16"], false);

  100.     rc = di.GetNextAsMap(&tensor_map);

  101.     EXPECT_TRUE(rc.IsOk());

  102.     if (tensor_map.size() == 0) {

  103.       success = true;

  104.     }

  105.   }

  106.   EXPECT_EQ(success, true);

  107. }

  108. 

  109. TEST_F(MindDataTestBatchOp, TestRepeatBatchDropTrue) {

  110.   std::string schema_file = datasets_root_path_ + "/testBatchDataset/test.data";

  111.   bool success = false;

  112.   auto tree = Build({TFReader(schema_file), Repeat(2), Batch(7, true, 99)});

  113.   tree->Prepare();

  114.   Status rc = tree->Launch();

  115.   if (rc.IsError()) {

  116.     MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";

  117.   } else {

  118.     int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807,

  119.                          -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807};

  120.     de::DatasetIterator di(tree);

  121.     std::shared_ptr<de::Tensor> t1, t2, t3;

  122.     rc = de::Tensor::CreateTensor(&t1, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  123.                                   (unsigned char *)payload);

  124.     EXPECT_TRUE(rc.IsOk());

  125.     rc = de::Tensor::CreateTensor(&t2, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  126.                                   (unsigned char *)(payload + 7));

  127.     EXPECT_TRUE(rc.IsOk());

  128.     rc = de::Tensor::CreateTensor(&t3, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  129.                                   (unsigned char *)(payload + 2));

  130.     EXPECT_TRUE(rc.IsOk());

  131. 

  132.     TensorMap tensor_map;

  133.     rc = di.GetNextAsMap(&tensor_map);

  134.     EXPECT_TRUE(rc.IsOk());

  135.     EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true);  // first call to getNext()

  136. 

  137.     rc = di.GetNextAsMap(&tensor_map);

  138.     EXPECT_TRUE(rc.IsOk());

  139.     EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true);  // second call to getNext()

  140. 

  141.     rc = di.GetNextAsMap(&tensor_map);

  142.     EXPECT_TRUE(rc.IsOk());

  143.     EXPECT_EQ(*t3 == *(tensor_map["col_sint64"]), true);  // third call to getNext()

  144. 

  145.     rc = di.GetNextAsMap(&tensor_map);

  146.     EXPECT_TRUE(rc.IsOk());

  147.     if (tensor_map.size() == 0) {

  148.       success = true;

  149.     }

  150.   }

  151.   EXPECT_EQ(success, true);

  152. }

  153. 

  154. TEST_F(MindDataTestBatchOp, TestRepeatBatchDropFalse) {

  155.   std::string schema_file = datasets_root_path_ + "/testBatchDataset/test.data";

  156.   bool success = false;

  157.   auto tree = Build({TFReader(schema_file), Repeat(2), Batch(7, false, 99)});

  158.   tree->Prepare();

  159.   Status rc = tree->Launch();

  160.   if (rc.IsError()) {

  161.     MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";

  162.   } else {

  163.     int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807,

  164.                          -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807};

  165.     de::DatasetIterator di(tree);

  166.     std::shared_ptr<de::Tensor> t1, t2, t3, t4;

  167.     rc = de::Tensor::CreateTensor(&t1, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  168.                                   (unsigned char *)payload);

  169.     EXPECT_TRUE(rc.IsOk());

  170.     rc = de::Tensor::CreateTensor(&t2, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  171.                                   (unsigned char *)(payload + 7));

  172.     EXPECT_TRUE(rc.IsOk());

  173.     rc = de::Tensor::CreateTensor(&t3, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  174.                                   (unsigned char *)(payload + 2));

  175.     EXPECT_TRUE(rc.IsOk());

  176.     rc = de::Tensor::CreateTensor(&t4, TensorImpl::kFlexible, de::TensorShape({3, 1}), de::DataType(DataType::DE_INT64),

  177.                                   (unsigned char *)(payload + 9));

  178.     EXPECT_TRUE(rc.IsOk());

  179. 

  180.     TensorMap tensor_map;

  181.     rc = di.GetNextAsMap(&tensor_map);

  182.     EXPECT_TRUE(rc.IsOk());

  183.     EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true);  // first call to getNext()

  184. 

  185.     rc = di.GetNextAsMap(&tensor_map);

  186.     EXPECT_TRUE(rc.IsOk());

  187.     EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true);  // second call to getNext()

  188. 

  189.     rc = di.GetNextAsMap(&tensor_map);

  190.     EXPECT_TRUE(rc.IsOk());

  191.     EXPECT_EQ(*t3 == *(tensor_map["col_sint64"]), true);  // third call to getNext()

  192. 

  193.     rc = di.GetNextAsMap(&tensor_map);

  194.     EXPECT_TRUE(rc.IsOk());

  195.     EXPECT_EQ(*t4 == *(tensor_map["col_sint64"]), true);  // last call to getNext()

  196. 

  197.     rc = di.GetNextAsMap(&tensor_map);

  198.     EXPECT_TRUE(rc.IsOk());

  199.     if (tensor_map.size() == 0) {

  200.       success = true;

  201.     }

  202.   }

  203.   EXPECT_EQ(success, true);

  204. }

  205. 

  206. TEST_F(MindDataTestBatchOp, TestBatchDropFalseRepeat) {

  207.   std::string schema_file = datasets_root_path_ + "/testBatchDataset/test.data";

  208.   bool success = false;

  209.   auto tree = Build({TFReader(schema_file), Batch(7, false, 99), Repeat(2)});

  210.   tree->Prepare();

  211.   Status rc = tree->Launch();

  212.   if (rc.IsError()) {

  213.     MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";

  214.   } else {

  215.     int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807,

  216.                          -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807};

  217.     de::DatasetIterator di(tree);

  218.     std::shared_ptr<de::Tensor> t1, t2;

  219.     rc = de::Tensor::CreateTensor(&t1, TensorImpl::kFlexible, de::TensorShape({7, 1}), de::DataType(DataType::DE_INT64),

  220.                                   (unsigned char *)payload);

  221.     EXPECT_TRUE(rc.IsOk());

  222.     rc = de::Tensor::CreateTensor(&t2, TensorImpl::kFlexible, de::TensorShape({5, 1}), de::DataType(DataType::DE_INT64),

  223.                                   (unsigned char *)(payload + 7));

  224.     EXPECT_TRUE(rc.IsOk());

  225. 

  226.     TensorMap tensor_map;

  227.     rc = di.GetNextAsMap(&tensor_map);

  228.     EXPECT_TRUE(rc.IsOk());

  229.     EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true);  // first call to getNext()

  230. 

  231.     rc = di.GetNextAsMap(&tensor_map);

  232.     EXPECT_TRUE(rc.IsOk());

  233.     EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true);  // second call to getNext()

  234. 

  235.     rc = di.GetNextAsMap(&tensor_map);

  236.     EXPECT_TRUE(rc.IsOk());

  237.     EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true);  // third call to getNext()

  238. 

  239.     rc = di.GetNextAsMap(&tensor_map);

  240.     EXPECT_TRUE(rc.IsOk());

  241.     EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true);  // last call to getNext()

  242. 

  243.     rc = di.GetNextAsMap(&tensor_map);

  244.     EXPECT_TRUE(rc.IsOk());

  245.     if (tensor_map.size() == 0) {

  246.       success = true;

  247.     }

  248.   }

  249.   EXPECT_EQ(success, true);

  250. }

  251. 

  252. TEST_F(MindDataTestBatchOp, TestBatchDropTrueRepeat) {

  253.   std::string schema_file = datasets_root_path_ + "/testBatchDataset/test.data";

  254.   bool success = false;

  255.   auto tree = Build({TFReader(schema_file), Batch(5, true, 99), Repeat(2)});

  256.   tree->Prepare();

  257.   Status rc = tree->Launch();

  258.   if (rc.IsError()) {

  259.     MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";

  260.   } else {

  261.     int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807,

  262.                          -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807};

  263.     de::DatasetIterator di(tree);

  264.     std::shared_ptr<de::Tensor> t1, t2;

  265.     rc = de::Tensor::CreateTensor(&t1, TensorImpl::kFlexible, de::TensorShape({5, 1}), de::DataType(DataType::DE_INT64),

  266.                                   (unsigned char *)payload);

  267.     EXPECT_TRUE(rc.IsOk());

  268.     rc = de::Tensor::CreateTensor(&t2, TensorImpl::kFlexible, de::TensorShape({5, 1}), de::DataType(DataType::DE_INT64),

  269.                                   (unsigned char *)(payload + 5));

  270.     EXPECT_TRUE(rc.IsOk());

  271. 

  272.     TensorMap tensor_map;

  273.     rc = di.GetNextAsMap(&tensor_map);

  274.     EXPECT_TRUE(rc.IsOk());

  275.     EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true);  // first call to getNext()

  276. 

  277.     rc = di.GetNextAsMap(&tensor_map);

  278.     EXPECT_TRUE(rc.IsOk());

  279.     EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true);  // second call to getNext()

  280. 

  281.     rc = di.GetNextAsMap(&tensor_map);

  282.     EXPECT_TRUE(rc.IsOk());

  283.     EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true);  // third call to getNext()

  284. 

  285.     rc = di.GetNextAsMap(&tensor_map);

  286.     EXPECT_TRUE(rc.IsOk());

  287.     EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true);  // last call to getNext()

  288. 

  289.     rc = di.GetNextAsMap(&tensor_map);

  290.     EXPECT_TRUE(rc.IsOk());

  291.     if (tensor_map.size() == 0) {

  292.       success = true;

  293.     }

  294.   }

  295.   EXPECT_EQ(success, true);

  296. }

  297. 

  298. TEST_F(MindDataTestBatchOp, TestSimpleBatchPadding) {

  299.   std::string schema_file = datasets_root_path_ + "/testBatchDataset/test.data";

  300.   std::shared_ptr<BatchOp> op;

  301.   PadInfo m;

  302.   std::shared_ptr<Tensor> pad_value;

  303.   Tensor::CreateTensor(&pad_value, TensorImpl::kFlexible, TensorShape::CreateScalar(), DataType(DataType::DE_FLOAT32));

  304.   pad_value->SetItemAt<float>({}, -1);

  305.   m.insert({"col_1d", std::make_pair(TensorShape({4}), pad_value)});

  306.   de::BatchOp::Builder(12).SetDrop(false).SetPaddingMap(m, true).Build(&op);

  307.   auto tree = Build({TFReader(schema_file), op});

  308.   tree->Prepare();

  309.   Status rc = tree->Launch();

  310.   if (rc.IsError()) {

  311.     MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";

  312.   } else {

  313.     int64_t payload[] = {-9223372036854775807 - 1,

  314.                          1,

  315.                          -1,

  316.                          -1,

  317.                          2,

  318.                          3,

  319.                          -1,

  320.                          -1,

  321.                          4,

  322.                          5,

  323.                          -1,

  324.                          -1,

  325.                          6,

  326.                          7,

  327.                          -1,

  328.                          -1,

  329.                          8,

  330.                          9,

  331.                          -1,

  332.                          -1,

  333.                          10,

  334.                          11,

  335.                          -1,

  336.                          -1,

  337.                          12,

  338.                          13,

  339.                          -1,

  340.                          -1,

  341.                          14,

  342.                          15,

  343.                          -1,

  344.                          -1,

  345.                          16,

  346.                          17,

  347.                          -1,

  348.                          -1,

  349.                          18,

  350.                          19,

  351.                          -1,

  352.                          -1,

  353.                          20,

  354.                          21,

  355.                          -1,

  356.                          -1,

  357.                          22,

  358.                          23,

  359.                          -1,

  360.                          -1};

  361.     std::shared_ptr<de::Tensor> t;

  362.     rc = de::Tensor::CreateTensor(&t, TensorImpl::kFlexible, de::TensorShape({12, 4}), de::DataType(DataType::DE_INT64),

  363.                                   (unsigned char *)payload);

  364.     de::DatasetIterator di(tree);

  365.     TensorMap tensor_map;

  366.     rc = di.GetNextAsMap(&tensor_map);

  367.     EXPECT_TRUE((*t) == (*(tensor_map["col_1d"])));

  368.     rc = di.GetNextAsMap(&tensor_map);

  369.     EXPECT_TRUE(tensor_map.size() == 0);

  370.     EXPECT_TRUE(rc.IsOk());

  371.   }

  372. }