diff --git a/mindspore/ccsrc/minddata/dataset/api/execute.cc b/mindspore/ccsrc/minddata/dataset/api/execute.cc index 03246b4496..8da53b8d5a 100644 --- a/mindspore/ccsrc/minddata/dataset/api/execute.cc +++ b/mindspore/ccsrc/minddata/dataset/api/execute.cc @@ -83,10 +83,9 @@ Execute::Execute(std::shared_ptr op, MapTargetDevice deviceType #endif } -/* -Execute::Execute(TensorTransform op, MapTargetDevice deviceType) { +Execute::Execute(std::reference_wrapper op, MapTargetDevice deviceType) { // Convert op from TensorTransform to TensorOperation - std::shared_ptr operation = op.Parse(); + std::shared_ptr operation = op.get().Parse(); ops_.emplace_back(std::move(operation)); device_type_ = deviceType; #ifdef ENABLE_ACL @@ -100,7 +99,6 @@ Execute::Execute(TensorTransform op, MapTargetDevice deviceType) { } #endif } -*/ // Execute function for the example case: auto decode(new vision::Decode()); Execute::Execute(TensorTransform *op, MapTargetDevice deviceType) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache.h b/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache.h index f57673a9b2..cea9fb70a0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache.h @@ -19,7 +19,7 @@ #include #include "minddata/dataset/engine/datasetops/dataset_op.h" -#include "minddata/dataset/include/samplers.h" +#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h" #include "minddata/dataset/util/status.h" namespace mindspore::dataset { diff --git a/mindspore/ccsrc/minddata/dataset/include/execute.h b/mindspore/ccsrc/minddata/dataset/include/execute.h index ee1e8c8430..d96b5db04a 100644 --- a/mindspore/ccsrc/minddata/dataset/include/execute.h +++ b/mindspore/ccsrc/minddata/dataset/include/execute.h @@ -36,7 +36,7 @@ class Execute { // FIXME - Temporarily overload Execute to support both TensorOperation and TensorTransform explicit Execute(std::shared_ptr op, MapTargetDevice deviceType = MapTargetDevice::kCpu); explicit Execute(std::shared_ptr op, MapTargetDevice deviceType = MapTargetDevice::kCpu); - // explicit Execute(TensorTransform op, MapTargetDevice deviceType = MapTargetDevice::KCpu); + explicit Execute(std::reference_wrapper op, MapTargetDevice deviceType = MapTargetDevice::kCpu); explicit Execute(TensorTransform *op, MapTargetDevice deviceType = MapTargetDevice::kCpu); explicit Execute(std::vector> ops, diff --git a/mindspore/ccsrc/minddata/dataset/include/samplers.h b/mindspore/ccsrc/minddata/dataset/include/samplers.h index 6dfe5fecfb..9603a5ed56 100644 --- a/mindspore/ccsrc/minddata/dataset/include/samplers.h +++ b/mindspore/ccsrc/minddata/dataset/include/samplers.h @@ -20,11 +20,12 @@ #include #include -// FIXME - This internal IR header will be removed when external API classes are provided -#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h" - namespace mindspore { namespace dataset { + +// Forward declare +class SamplerObj; + // Abstract class to represent a sampler in the data pipeline. /// \class Sampler samplers.h /// \brief An abstract base class to represent a sampler in the data pipeline. diff --git a/tests/ut/cpp/dataset/c_api_samplers_test.cc b/tests/ut/cpp/dataset/c_api_samplers_test.cc index e0060eae89..7541bb5b44 100644 --- a/tests/ut/cpp/dataset/c_api_samplers_test.cc +++ b/tests/ut/cpp/dataset/c_api_samplers_test.cc @@ -15,6 +15,7 @@ */ #include "common/common.h" #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" +#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h" #include "minddata/dataset/include/datasets.h" #include diff --git a/tests/ut/cpp/dataset/execute_test.cc b/tests/ut/cpp/dataset/execute_test.cc index 474cd3beac..4d138d12df 100644 --- a/tests/ut/cpp/dataset/execute_test.cc +++ b/tests/ut/cpp/dataset/execute_test.cc @@ -13,8 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include "common/common.h" #include "common/cvop_common.h" +#include "include/api/types.h" #include "minddata/dataset/core/de_tensor.h" #include "minddata/dataset/include/execute.h" #include "minddata/dataset/include/transforms.h" @@ -33,12 +35,37 @@ class MindDataTestExecute : public UT::CVOP::CVOpCommon { std::shared_ptr output_tensor_; }; +mindspore::MSTensor ReadFileToTensor(const std::string &file) { + if (file.empty()) { + MS_LOG(ERROR) << "Pointer file is nullptr, return an empty Tensor."; + return mindspore::MSTensor(); + } + std::ifstream ifs(file); + if (!ifs.good()) { + MS_LOG(ERROR) << "File: " << file << " does not exist, return an empty Tensor."; + return mindspore::MSTensor(); + } + if (!ifs.is_open()) { + MS_LOG(ERROR) << "File: " << file << "open failed, return an empty Tensor."; + return mindspore::MSTensor(); + } + + ifs.seekg(0, std::ios::end); + size_t size = ifs.tellg(); + mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast(size)}, nullptr, size); + + ifs.seekg(0, std::ios::beg); + ifs.read(reinterpret_cast(buf.MutableData()), size); + ifs.close(); + + return buf; +} + TEST_F(MindDataTestExecute, TestComposeTransforms) { MS_LOG(INFO) << "Doing TestComposeTransforms."; - std::shared_ptr de_tensor; - mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor); - auto image = mindspore::MSTensor(std::make_shared(de_tensor)); + // Read images + auto image = ReadFileToTensor("data/dataset/apple.jpg"); // Transform params std::shared_ptr decode = std::make_shared(); @@ -59,9 +86,7 @@ TEST_F(MindDataTestExecute, TestTransformInput1) { // instantiated via mix of make_shared and new // Read images - std::shared_ptr de_tensor; - mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor); - auto image = mindspore::MSTensor(std::make_shared(de_tensor)); + auto image = ReadFileToTensor("data/dataset/apple.jpg"); // Define transform operations std::shared_ptr decode = std::make_shared(); @@ -88,7 +113,7 @@ TEST_F(MindDataTestExecute, TestTransformInput2) { // Test Execute with transform op input using API constructors, with std::shared_ptr de_tensor; mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor); auto image = mindspore::MSTensor(std::make_shared(de_tensor)); @@ -117,30 +142,30 @@ TEST_F(MindDataTestExecute, TestTransformInput3) { MS_LOG(INFO) << "Doing MindDataTestExecute-TestTransformInput3."; // Test Execute with transform op input using API constructors, with auto pointers - // Read image + // Read image, construct MSTensor from dataset tensor std::shared_ptr de_tensor; mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor); auto image = mindspore::MSTensor(std::make_shared(de_tensor)); // Define transform operations - auto decode(new vision::Decode()); // auto will create raw pointer to Decode class - auto resize(new vision::Resize({224, 224})); - auto normalize( - new vision::Normalize({0.485 * 255, 0.456 * 255, 0.406 * 255}, {0.229 * 255, 0.224 * 255, 0.225 * 255})); - auto hwc2chw(new vision::HWC2CHW()); + auto decode = vision::Decode(); + mindspore::dataset::Execute Transform1(decode); - std::vector op_list = {decode, resize, normalize, hwc2chw}; - mindspore::dataset::Execute Transform(op_list); + auto resize = vision::Resize({224, 224}); + mindspore::dataset::Execute Transform2(resize); // Apply transform on image - Status rc = Transform(image, &image); + Status rc; + rc = Transform1(image, &image); + ASSERT_TRUE(rc.IsOk()); + rc = Transform2(image, &image); + ASSERT_TRUE(rc.IsOk()); // Check image info - ASSERT_TRUE(rc.IsOk()); ASSERT_EQ(image.Shape().size(), 3); - ASSERT_EQ(image.Shape()[0], 3); + ASSERT_EQ(image.Shape()[0], 224); ASSERT_EQ(image.Shape()[1], 224); - ASSERT_EQ(image.Shape()[2], 224); + ASSERT_EQ(image.Shape()[2], 3); } TEST_F(MindDataTestExecute, TestTransformInputSequential) { @@ -148,28 +173,26 @@ TEST_F(MindDataTestExecute, TestTransformInputSequential) { // Test Execute with transform op input using API constructors, with auto pointers; // Apply 2 transformations sequentially, including single non-vector Transform op input - // Read images + // Read image, construct MSTensor from dataset tensor std::shared_ptr de_tensor; mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor); auto image = mindspore::MSTensor(std::make_shared(de_tensor)); // Define transform#1 operations - auto decode(new vision::Decode()); // auto will create raw pointer to Decode class - auto resize(new vision::Resize({224, 224})); - auto normalize( + std::shared_ptr decode(new vision::Decode()); + std::shared_ptr resize(new vision::Resize({224, 224})); + std::shared_ptr normalize( new vision::Normalize({0.485 * 255, 0.456 * 255, 0.406 * 255}, {0.229 * 255, 0.224 * 255, 0.225 * 255})); - std::vector op_list = {decode, resize, normalize}; + std::vector> op_list = {decode, resize, normalize}; mindspore::dataset::Execute Transform(op_list); // Apply transform#1 on image Status rc = Transform(image, &image); // Define transform#2 operations - auto hwc2chw(new vision::HWC2CHW()); - - TensorTransform *op_single = hwc2chw; - mindspore::dataset::Execute Transform2(op_single); + std::shared_ptr hwc2chw(new vision::HWC2CHW()); + mindspore::dataset::Execute Transform2(hwc2chw); // Apply transform#2 on image rc = Transform2(image, &image); @@ -186,7 +209,7 @@ TEST_F(MindDataTestExecute, TestTransformDecodeResizeCenterCrop1) { MS_LOG(INFO) << "Doing MindDataTestExecute-TestTransformDecodeResizeCenterCrop1."; // Test Execute with Decode, Resize and CenterCrop transform ops input using API constructors, with auto pointers - // Read image + // Read image, construct MSTensor from dataset tensor std::shared_ptr de_tensor; mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor); auto image = mindspore::MSTensor(std::make_shared(de_tensor)); @@ -194,12 +217,12 @@ TEST_F(MindDataTestExecute, TestTransformDecodeResizeCenterCrop1) { // Define transform operations std::vector resize_paras = {256, 256}; std::vector crop_paras = {224, 224}; - auto decode(new vision::Decode()); - auto resize(new vision::Resize(resize_paras)); - auto centercrop(new vision::CenterCrop(crop_paras)); - auto hwc2chw(new vision::HWC2CHW()); + std::shared_ptr decode(new vision::Decode()); + std::shared_ptr resize(new vision::Resize(resize_paras)); + std::shared_ptr centercrop(new vision::CenterCrop(crop_paras)); + std::shared_ptr hwc2chw(new vision::HWC2CHW()); - std::vector op_list = {decode, resize, centercrop, hwc2chw}; + std::vector> op_list = {decode, resize, centercrop, hwc2chw}; mindspore::dataset::Execute Transform(op_list, MapTargetDevice::kCpu); // Apply transform on image