From 82103a693d05e613f15484974148adbc63e1bb7f Mon Sep 17 00:00:00 2001 From: nhussain Date: Tue, 21 Jul 2020 15:44:13 -0400 Subject: [PATCH] restructuring --- .../ccsrc/minddata/dataset/api/CMakeLists.txt | 35 +- .../minddata/dataset/api/python/bindings.cc | 122 ++ .../python/bindings/dataset/core/bindings.cc | 114 ++ .../bindings/dataset/engine/cache/bindings.cc | 29 + .../dataset/engine/datasetops/bindings.cc | 35 + .../engine/datasetops/source/bindings.cc | 186 +++ .../datasetops/source/sampler/bindings.cc | 88 ++ .../bindings/dataset/engine/gnn/bindings.cc | 101 ++ .../bindings/dataset/kernels/bindings.cc | 91 ++ .../bindings/dataset/kernels/data/bindings.cc | 133 ++ .../dataset/kernels/image/bindings.cc | 346 +++++ .../python/bindings/dataset/text/bindings.cc | 92 ++ .../bindings/dataset/text/kernels/bindings.cc | 244 ++++ .../bindings/mindrecord/include/bindings.cc | 87 ++ .../dataset/api/{ => python}/de_pipeline.cc | 2 +- .../dataset/api/{ => python}/de_pipeline.h | 0 .../dataset/api/python/pybind_register.cc | 40 + .../dataset/api/python/pybind_register.h | 81 ++ .../minddata/dataset/api/python_bindings.cc | 1138 ----------------- .../minddata/dataset/kernels/CMakeLists.txt | 12 +- .../dataset/kernels/{ => data}/compose_op.cc | 2 +- .../dataset/kernels/{ => data}/compose_op.h | 0 .../dataset/kernels/{ => data}/no_op.h | 0 .../kernels/{ => data}/random_apply_op.cc | 2 +- .../kernels/{ => data}/random_apply_op.h | 2 +- .../kernels/{ => data}/random_choice_op.cc | 2 +- .../kernels/{ => data}/random_choice_op.h | 2 +- .../image/random_vertical_flip_with_bbox_op.h | 1 - 28 files changed, 1825 insertions(+), 1162 deletions(-) create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/cache/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/gnn/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/data/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/image/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/bindings.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/bindings/mindrecord/include/bindings.cc rename mindspore/ccsrc/minddata/dataset/api/{ => python}/de_pipeline.cc (99%) rename mindspore/ccsrc/minddata/dataset/api/{ => python}/de_pipeline.h (100%) create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/pybind_register.cc create mode 100644 mindspore/ccsrc/minddata/dataset/api/python/pybind_register.h delete mode 100644 mindspore/ccsrc/minddata/dataset/api/python_bindings.cc rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/compose_op.cc (97%) rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/compose_op.h (100%) rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/no_op.h (100%) rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/random_apply_op.cc (97%) rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/random_apply_op.h (98%) rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/random_choice_op.cc (98%) rename mindspore/ccsrc/minddata/dataset/kernels/{ => data}/random_choice_op.h (98%) diff --git a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt index ae0b9cc28e..1f1c3c9bdd 100644 --- a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt @@ -1,16 +1,29 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) if (ENABLE_PYTHON) - add_library(APItoPython OBJECT - de_pipeline.cc - python_bindings.cc - ) - target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS}) -endif() + add_library(APItoPython OBJECT + python/de_pipeline.cc + python/pybind_register.cc + python/bindings.cc + python/bindings/dataset/engine/cache/bindings.cc + python/bindings/dataset/core/bindings.cc + python/bindings/dataset/kernels/data/bindings.cc + python/bindings/dataset/kernels/bindings.cc + python/bindings/dataset/engine/datasetops/bindings.cc + python/bindings/dataset/engine/datasetops/source/bindings.cc + python/bindings/dataset/engine/gnn/bindings.cc + python/bindings/dataset/kernels/image/bindings.cc + python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc + python/bindings/dataset/text/bindings.cc + python/bindings/dataset/text/kernels/bindings.cc + python/bindings/mindrecord/include/bindings.cc + ) + target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS}) +endif () add_library(cpp-API OBJECT - datasets.cc - iterator.cc - transforms.cc - samplers.cc - ) + datasets.cc + iterator.cc + transforms.cc + samplers.cc + ) diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings.cc new file mode 100644 index 0000000000..c578c295e5 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings.cc @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/api/python/de_pipeline.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER( + DEPipeline, 0, ([](const py::module *m) { + (void)py::class_(*m, "DEPipeline") + .def(py::init<>()) + .def( + "AddNodeToTree", + [](DEPipeline &de, const OpName &op_name, const py::dict &args) { + py::dict out; + THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &out)); + return out; + }, + py::return_value_policy::reference) + .def_static("AddChildToParentNode", + [](const DsOpPtr &child_op, const DsOpPtr &parent_op) { + THROW_IF_ERROR(DEPipeline::AddChildToParentNode(child_op, parent_op)); + }) + .def("AssignRootNode", + [](DEPipeline &de, const DsOpPtr &dataset_op) { THROW_IF_ERROR(de.AssignRootNode(dataset_op)); }) + .def("SetBatchParameters", + [](DEPipeline &de, const py::dict &args) { THROW_IF_ERROR(de.SetBatchParameters(args)); }) + .def("LaunchTreeExec", [](DEPipeline &de, int32_t num_epochs) { THROW_IF_ERROR(de.LaunchTreeExec(num_epochs)); }) + .def("GetNextAsMap", + [](DEPipeline &de) { + py::dict out; + THROW_IF_ERROR(de.GetNextAsMap(&out)); + return out; + }) + .def("GetNextAsList", + [](DEPipeline &de) { + py::list out; + THROW_IF_ERROR(de.GetNextAsList(&out)); + return out; + }) + .def("GetOutputShapes", + [](DEPipeline &de) { + py::list out; + THROW_IF_ERROR(de.GetOutputShapes(&out)); + return out; + }) + .def("GetOutputTypes", + [](DEPipeline &de) { + py::list out; + THROW_IF_ERROR(de.GetOutputTypes(&out)); + return out; + }) + .def("GetDatasetSize", &DEPipeline::GetDatasetSize) + .def("GetBatchSize", &DEPipeline::GetBatchSize) + .def("GetNumClasses", &DEPipeline::GetNumClasses) + .def("GetRepeatCount", &DEPipeline::GetRepeatCount) + .def("StopSend", [](DEPipeline &de) { THROW_IF_ERROR(de.StopSend()); }) + .def("SaveDataset", [](DEPipeline &de, const std::vector &file_names, const std::string &file_type) { + THROW_IF_ERROR(de.SaveDataset(file_names, file_type)); + return true; + }); + })); + +PYBIND_REGISTER(OpName, 0, ([](const py::module *m) { + (void)py::enum_(*m, "OpName", py::arithmetic()) + .value("SHUFFLE", OpName::kShuffle) + .value("BATCH", OpName::kBatch) + .value("BUCKETBATCH", OpName::kBucketBatch) + .value("BARRIER", OpName::kBarrier) + .value("MINDRECORD", OpName::kMindrecord) + .value("CACHE", OpName::kCache) + .value("REPEAT", OpName::kRepeat) + .value("SKIP", OpName::kSkip) + .value("TAKE", OpName::kTake) + .value("ZIP", OpName::kZip) + .value("CONCAT", OpName::kConcat) + .value("MAP", OpName::kMap) + .value("FILTER", OpName::kFilter) + .value("DEVICEQUEUE", OpName::kDeviceQueue) + .value("GENERATOR", OpName::kGenerator) + .export_values() + .value("RENAME", OpName::kRename) + .value("TFREADER", OpName::kTfReader) + .value("PROJECT", OpName::kProject) + .value("IMAGEFOLDER", OpName::kImageFolder) + .value("MNIST", OpName::kMnist) + .value("MANIFEST", OpName::kManifest) + .value("VOC", OpName::kVoc) + .value("COCO", OpName::kCoco) + .value("CIFAR10", OpName::kCifar10) + .value("CIFAR100", OpName::kCifar100) + .value("RANDOMDATA", OpName::kRandomData) + .value("BUILDVOCAB", OpName::kBuildVocab) + .value("SENTENCEPIECEVOCAB", OpName::kSentencePieceVocab) + .value("CELEBA", OpName::kCelebA) + .value("TEXTFILE", OpName::kTextFile) + .value("EPOCHCTRL", OpName::kEpochCtrl) + .value("CSV", OpName::kCsv) + .value("CLUE", OpName::kClue); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc new file mode 100644 index 0000000000..affc3600df --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/core/global_context.h" +#include "minddata/dataset/core/constants.h" +#include "minddata/dataset/api/python/de_pipeline.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(GlobalContext, 0, ([](const py::module *m) { + (void)py::class_(*m, "GlobalContext") + .def_static("config_manager", &GlobalContext::config_manager, py::return_value_policy::reference); + })); + +PYBIND_REGISTER(ConfigManager, 0, ([](const py::module *m) { + (void)py::class_>(*m, "ConfigManager") + .def("__str__", &ConfigManager::ToString) + .def("set_rows_per_buffer", &ConfigManager::set_rows_per_buffer) + .def("set_num_parallel_workers", &ConfigManager::set_num_parallel_workers) + .def("set_worker_connector_size", &ConfigManager::set_worker_connector_size) + .def("set_op_connector_size", &ConfigManager::set_op_connector_size) + .def("set_seed", &ConfigManager::set_seed) + .def("set_monitor_sampling_interval", &ConfigManager::set_monitor_sampling_interval) + .def("get_rows_per_buffer", &ConfigManager::rows_per_buffer) + .def("get_num_parallel_workers", &ConfigManager::num_parallel_workers) + .def("get_worker_connector_size", &ConfigManager::worker_connector_size) + .def("get_op_connector_size", &ConfigManager::op_connector_size) + .def("get_seed", &ConfigManager::seed) + .def("get_monitor_sampling_interval", &ConfigManager::monitor_sampling_interval) + .def("load", [](ConfigManager &c, std::string s) { THROW_IF_ERROR(c.LoadFile(s)); }); + })); + +PYBIND_REGISTER(Tensor, 0, ([](const py::module *m) { + (void)py::class_>(*m, "Tensor", py::buffer_protocol()) + .def(py::init([](py::array arr) { + std::shared_ptr out; + THROW_IF_ERROR(Tensor::CreateFromNpArray(arr, &out)); + return out; + })) + .def_buffer([](Tensor &tensor) { + py::buffer_info info; + THROW_IF_ERROR(Tensor::GetBufferInfo(&tensor, &info)); + return info; + }) + .def("__str__", &Tensor::ToString) + .def("shape", &Tensor::shape) + .def("type", &Tensor::type) + .def("as_array", [](py::object &t) { + auto &tensor = py::cast(t); + if (tensor.type() == DataType::DE_STRING) { + py::array res; + tensor.GetDataAsNumpyStrings(&res); + return res; + } + py::buffer_info info; + THROW_IF_ERROR(Tensor::GetBufferInfo(&tensor, &info)); + return py::array(pybind11::dtype(info), info.shape, info.strides, info.ptr, t); + }); + })); + +PYBIND_REGISTER(TensorShape, 0, ([](const py::module *m) { + (void)py::class_(*m, "TensorShape") + .def(py::init()) + .def("__str__", &TensorShape::ToString) + .def("as_list", &TensorShape::AsPyList) + .def("is_known", &TensorShape::known); + })); + +PYBIND_REGISTER(DataType, 0, ([](const py::module *m) { + (void)py::class_(*m, "DataType") + .def(py::init()) + .def(py::self == py::self) + .def("__str__", &DataType::ToString) + .def("__deepcopy__", [](py::object &t, py::dict memo) { return t; }); + })); + +PYBIND_REGISTER(BorderType, 0, ([](const py::module *m) { + (void)py::enum_(*m, "BorderType", py::arithmetic()) + .value("DE_BORDER_CONSTANT", BorderType::kConstant) + .value("DE_BORDER_EDGE", BorderType::kEdge) + .value("DE_BORDER_REFLECT", BorderType::kReflect) + .value("DE_BORDER_SYMMETRIC", BorderType::kSymmetric) + .export_values(); + })); + +PYBIND_REGISTER(InterpolationMode, 0, ([](const py::module *m) { + (void)py::enum_(*m, "InterpolationMode", py::arithmetic()) + .value("DE_INTER_LINEAR", InterpolationMode::kLinear) + .value("DE_INTER_CUBIC", InterpolationMode::kCubic) + .value("DE_INTER_AREA", InterpolationMode::kArea) + .value("DE_INTER_NEAREST_NEIGHBOUR", InterpolationMode::kNearestNeighbour) + .export_values(); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/cache/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/cache/bindings.cc new file mode 100644 index 0000000000..aa5ba9e561 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/cache/bindings.cc @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/engine/cache/cache_client.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(CacheClient, 0, ([](const py::module *m) { + (void)py::class_>(*m, "CacheClient") + .def(py::init()); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/bindings.cc new file mode 100644 index 0000000000..54844b8a2a --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/bindings.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/engine/datasetops/batch_op.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(CBatchInfo, 0, ([](const py::module *m) { + (void)py::class_(*m, "CBatchInfo") + .def(py::init()) + .def("get_epoch_num", &BatchOp::CBatchInfo::get_epoch_num) + .def("get_batch_num", &BatchOp::CBatchInfo::get_batch_num); + })); + +PYBIND_REGISTER(DatasetOp, 0, ([](const py::module *m) { + (void)py::class_>(*m, "DatasetOp"); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc new file mode 100644 index 0000000000..a44e538e37 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/bindings.cc @@ -0,0 +1,186 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/api/python/pybind_register.h" + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/engine/datasetops/dataset_op.h" + +#include "minddata/dataset/engine/datasetops/source/cifar_op.h" +#include "minddata/dataset/engine/datasetops/source/clue_op.h" +#include "minddata/dataset/engine/datasetops/source/csv_op.h" +#include "minddata/dataset/engine/datasetops/source/coco_op.h" +#include "minddata/dataset/engine/datasetops/source/image_folder_op.h" +#include "minddata/dataset/engine/datasetops/source/io_block.h" +#include "minddata/dataset/engine/datasetops/source/manifest_op.h" +#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h" +#include "minddata/dataset/engine/datasetops/source/mnist_op.h" +#include "minddata/dataset/engine/datasetops/source/random_data_op.h" +#include "minddata/dataset/engine/datasetops/source/text_file_op.h" +#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" +#include "minddata/dataset/engine/datasetops/source/voc_op.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(CifarOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "CifarOp") + .def_static("get_num_rows", [](const std::string &dir, bool isCifar10) { + int64_t count = 0; + THROW_IF_ERROR(CifarOp::CountTotalRows(dir, isCifar10, &count)); + return count; + }); + })); + +PYBIND_REGISTER(ClueOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "ClueOp") + .def_static("get_num_rows", [](const py::list &files) { + int64_t count = 0; + std::vector filenames; + for (auto file : files) { + file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file)); + } + THROW_IF_ERROR(ClueOp::CountAllFileRows(filenames, &count)); + return count; + }); + })); + +PYBIND_REGISTER(CsvOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "CsvOp") + .def_static("get_num_rows", [](const py::list &files, bool csv_header) { + int64_t count = 0; + std::vector filenames; + for (auto file : files) { + file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file)); + } + THROW_IF_ERROR(CsvOp::CountAllFileRows(filenames, csv_header, &count)); + return count; + }); + })); +PYBIND_REGISTER(CocoOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "CocoOp") + .def_static("get_class_indexing", + [](const std::string &dir, const std::string &file, const std::string &task) { + std::vector>> output_class_indexing; + THROW_IF_ERROR(CocoOp::GetClassIndexing(dir, file, task, &output_class_indexing)); + return output_class_indexing; + }) + .def_static("get_num_rows", + [](const std::string &dir, const std::string &file, const std::string &task) { + int64_t count = 0; + THROW_IF_ERROR(CocoOp::CountTotalRows(dir, file, task, &count)); + return count; + }); + })); + +PYBIND_REGISTER(ImageFolderOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "ImageFolderOp") + .def_static("get_num_rows_and_classes", [](const std::string &path) { + int64_t count = 0, num_classes = 0; + THROW_IF_ERROR( + ImageFolderOp::CountRowsAndClasses(path, std::set{}, &count, &num_classes)); + return py::make_tuple(count, num_classes); + }); + })); + +PYBIND_REGISTER(ManifestOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "ManifestOp") + .def_static("get_num_rows_and_classes", + [](const std::string &file, const py::dict &dict, const std::string &usage) { + int64_t count = 0, num_classes = 0; + THROW_IF_ERROR(ManifestOp::CountTotalRows(file, dict, usage, &count, &num_classes)); + return py::make_tuple(count, num_classes); + }) + .def_static("get_class_indexing", [](const std::string &file, const py::dict &dict, + const std::string &usage) { + std::map output_class_indexing; + THROW_IF_ERROR(ManifestOp::GetClassIndexing(file, dict, usage, &output_class_indexing)); + return output_class_indexing; + }); + })); +PYBIND_REGISTER(MindRecordOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "MindRecordOp") + .def_static("get_num_rows", [](const std::vector &paths, bool load_dataset, + const py::object &sampler, const int64_t num_padded) { + int64_t count = 0; + std::shared_ptr op; + if (py::hasattr(sampler, "create_for_minddataset")) { + auto create = sampler.attr("create_for_minddataset"); + op = create().cast>(); + } + THROW_IF_ERROR(MindRecordOp::CountTotalRows(paths, load_dataset, op, &count, num_padded)); + return count; + }); + })); + +PYBIND_REGISTER(MnistOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "MnistOp") + .def_static("get_num_rows", [](const std::string &dir) { + int64_t count = 0; + THROW_IF_ERROR(MnistOp::CountTotalRows(dir, &count)); + return count; + }); + })); + +PYBIND_REGISTER(TextFileOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "TextFileOp") + .def_static("get_num_rows", [](const py::list &files) { + int64_t count = 0; + std::vector filenames; + for (auto file : files) { + !file.is_none() ? filenames.push_back(py::str(file)) : (void)filenames.emplace_back(""); + } + THROW_IF_ERROR(TextFileOp::CountAllFileRows(filenames, &count)); + return count; + }); + })); + +PYBIND_REGISTER(TFReaderOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "TFReaderOp") + .def_static( + "get_num_rows", [](const py::list &files, int64_t numParallelWorkers, bool estimate = false) { + int64_t count = 0; + std::vector filenames; + for (auto l : files) { + !l.is_none() ? filenames.push_back(py::str(l)) : (void)filenames.emplace_back(""); + } + THROW_IF_ERROR(TFReaderOp::CountTotalRows(&count, filenames, numParallelWorkers, estimate)); + return count; + }); + })); + +PYBIND_REGISTER(VOCOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "VOCOp") + .def_static("get_num_rows", + [](const std::string &dir, const std::string &task_type, const std::string &task_mode, + const py::dict &dict, int64_t numSamples) { + int64_t count = 0; + THROW_IF_ERROR(VOCOp::CountTotalRows(dir, task_type, task_mode, dict, &count)); + return count; + }) + .def_static("get_class_indexing", [](const std::string &dir, const std::string &task_type, + const std::string &task_mode, const py::dict &dict) { + std::map output_class_indexing; + THROW_IF_ERROR(VOCOp::GetClassIndexing(dir, task_type, task_mode, dict, &output_class_indexing)); + return output_class_indexing; + }); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc new file mode 100644 index 0000000000..29aa0f12d6 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/pk_sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/python_sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(Sampler, 0, ([](const py::module *m) { + (void)py::class_>(*m, "Sampler") + .def("set_num_rows", + [](Sampler &self, int64_t rows) { THROW_IF_ERROR(self.SetNumRowsInDataset(rows)); }) + .def("set_num_samples", + [](Sampler &self, int64_t samples) { THROW_IF_ERROR(self.SetNumSamples(samples)); }) + .def("initialize", [](Sampler &self) { THROW_IF_ERROR(self.InitSampler()); }) + .def("get_indices", + [](Sampler &self) { + py::array ret; + THROW_IF_ERROR(self.GetAllIdsThenReset(&ret)); + return ret; + }) + .def("add_child", [](std::shared_ptr self, std::shared_ptr child) { + THROW_IF_ERROR(self->AddChild(child)); + }); + })); + +PYBIND_REGISTER(DistributedSampler, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "DistributedSampler") + .def(py::init()); + })); + +PYBIND_REGISTER(PKSampler, 1, ([](const py::module *m) { + (void)py::class_>(*m, "PKSampler") + .def(py::init()); + })); + +PYBIND_REGISTER(PythonSampler, 1, ([](const py::module *m) { + (void)py::class_>(*m, "PythonSampler") + .def(py::init()); + })); + +PYBIND_REGISTER(RandomSampler, 1, ([](const py::module *m) { + (void)py::class_>(*m, "RandomSampler") + .def(py::init()); + })); + +PYBIND_REGISTER(SequentialSampler, 1, ([](const py::module *m) { + (void)py::class_>(*m, + "SequentialSampler") + .def(py::init()); + })); + +PYBIND_REGISTER(SubsetRandomSampler, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "SubsetRandomSampler") + .def(py::init>()); + })); + +PYBIND_REGISTER(WeightedRandomSampler, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "WeightedRandomSampler") + .def(py::init, bool>()); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/gnn/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/gnn/bindings.cc new file mode 100644 index 0000000000..18dcfb470a --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/gnn/bindings.cc @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" + +#include "minddata/dataset/engine/gnn/graph.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER( + Graph, 0, ([](const py::module *m) { + (void)py::class_>(*m, "Graph") + .def(py::init([](std::string dataset_file, int32_t num_workers) { + std::shared_ptr g_out = std::make_shared(dataset_file, num_workers); + THROW_IF_ERROR(g_out->Init()); + return g_out; + })) + .def("get_all_nodes", + [](gnn::Graph &g, gnn::NodeType node_type) { + std::shared_ptr out; + THROW_IF_ERROR(g.GetAllNodes(node_type, &out)); + return out; + }) + .def("get_all_edges", + [](gnn::Graph &g, gnn::EdgeType edge_type) { + std::shared_ptr out; + THROW_IF_ERROR(g.GetAllEdges(edge_type, &out)); + return out; + }) + .def("get_nodes_from_edges", + [](gnn::Graph &g, std::vector edge_list) { + std::shared_ptr out; + THROW_IF_ERROR(g.GetNodesFromEdges(edge_list, &out)); + return out; + }) + .def("get_all_neighbors", + [](gnn::Graph &g, std::vector node_list, gnn::NodeType neighbor_type) { + std::shared_ptr out; + THROW_IF_ERROR(g.GetAllNeighbors(node_list, neighbor_type, &out)); + return out; + }) + .def("get_sampled_neighbors", + [](gnn::Graph &g, std::vector node_list, std::vector neighbor_nums, + std::vector neighbor_types) { + std::shared_ptr out; + THROW_IF_ERROR(g.GetSampledNeighbors(node_list, neighbor_nums, neighbor_types, &out)); + return out; + }) + .def("get_neg_sampled_neighbors", + [](gnn::Graph &g, std::vector node_list, gnn::NodeIdType neighbor_num, + gnn::NodeType neg_neighbor_type) { + std::shared_ptr out; + THROW_IF_ERROR(g.GetNegSampledNeighbors(node_list, neighbor_num, neg_neighbor_type, &out)); + return out; + }) + .def("get_node_feature", + [](gnn::Graph &g, std::shared_ptr node_list, std::vector feature_types) { + TensorRow out; + THROW_IF_ERROR(g.GetNodeFeature(node_list, feature_types, &out)); + return out.getRow(); + }) + .def("get_edge_feature", + [](gnn::Graph &g, std::shared_ptr edge_list, std::vector feature_types) { + TensorRow out; + THROW_IF_ERROR(g.GetEdgeFeature(edge_list, feature_types, &out)); + return out.getRow(); + }) + .def("graph_info", + [](gnn::Graph &g) { + py::dict out; + THROW_IF_ERROR(g.GraphInfo(&out)); + return out; + }) + .def("random_walk", + [](gnn::Graph &g, std::vector node_list, std::vector meta_path, + float step_home_param, float step_away_param, gnn::NodeIdType default_node) { + std::shared_ptr out; + THROW_IF_ERROR(g.RandomWalk(node_list, meta_path, step_home_param, step_away_param, default_node, &out)); + return out; + }); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/bindings.cc new file mode 100644 index 0000000000..32e52aed42 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/bindings.cc @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pybind11/pybind11.h" + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/core/global_context.h" +#include "minddata/dataset/api/python/de_pipeline.h" + +#include "mindspore/ccsrc/minddata/dataset/kernels/data/compose_op.h" +#include "mindspore/ccsrc/minddata/dataset/kernels/data/no_op.h" +#include "minddata/dataset/kernels/py_func_op.h" +#include "mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.h" +#include "mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.h" + +namespace mindspore { +namespace dataset { + +Status PyListToTensorOps(const py::list &py_ops, std::vector> *ops) { + RETURN_UNEXPECTED_IF_NULL(ops); + for (auto op : py_ops) { + if (py::isinstance(op)) { + ops->emplace_back(op.cast>()); + } else if (py::isinstance(op)) { + ops->emplace_back(std::make_shared(op.cast())); + } else { + RETURN_STATUS_UNEXPECTED("element is neither a TensorOp nor a pyfunc."); + } + } + CHECK_FAIL_RETURN_UNEXPECTED(!ops->empty(), "TensorOp list is empty."); + for (auto const &op : *ops) { + RETURN_UNEXPECTED_IF_NULL(op); + } + return Status::OK(); +} + +PYBIND_REGISTER(TensorOp, 0, ([](const py::module *m) { + (void)py::class_>(*m, "TensorOp") + .def("__deepcopy__", [](py::object &t, py::dict memo) { return t; }); + })); + +PYBIND_REGISTER(ComposeOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "ComposeOp") + .def(py::init([](const py::list &ops) { + std::vector> t_ops; + THROW_IF_ERROR(PyListToTensorOps(ops, &t_ops)); + return std::make_shared(t_ops); + })); + })); + +PYBIND_REGISTER(NoOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "NoOp", "TensorOp that does nothing, for testing purposes only.") + .def(py::init<>()); + })); + +PYBIND_REGISTER(RandomChoiceOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "RandomChoiceOp") + .def(py::init([](const py::list &ops) { + std::vector> t_ops; + THROW_IF_ERROR(PyListToTensorOps(ops, &t_ops)); + return std::make_shared(t_ops); + })); + })); + +PYBIND_REGISTER(RandomApplyOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "RandomApplyOp") + .def(py::init([](double prob, const py::list &ops) { + std::vector> t_ops; + THROW_IF_ERROR(PyListToTensorOps(ops, &t_ops)); + if (prob < 0 || prob > 1) { + THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "prob needs to be within [0,1].")); + } + return std::make_shared(prob, t_ops); + })); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/data/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/data/bindings.cc new file mode 100644 index 0000000000..3500885dac --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/data/bindings.cc @@ -0,0 +1,133 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/kernels/data/concatenate_op.h" +#include "minddata/dataset/kernels/data/duplicate_op.h" +#include "minddata/dataset/kernels/data/fill_op.h" +#include "minddata/dataset/kernels/data/mask_op.h" +#include "minddata/dataset/kernels/data/one_hot_op.h" +#include "minddata/dataset/kernels/data/pad_end_op.h" +#include "minddata/dataset/kernels/data/slice_op.h" +#include "minddata/dataset/kernels/data/to_float16_op.h" +#include "minddata/dataset/kernels/data/type_cast_op.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(ConcatenateOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "ConcatenateOp", "Tensor operation concatenate tensors.") + .def(py::init, std::shared_ptr>(), py::arg("axis"), + py::arg("prepend").none(true), py::arg("append").none(true)); + })); + +PYBIND_REGISTER(DuplicateOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "DuplicateOp", + "Duplicate tensor.") + .def(py::init<>()); + })); + +PYBIND_REGISTER(FillOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "FillOp", "Tensor operation to return tensor filled with same value as input fill value.") + .def(py::init>()); + })); + +PYBIND_REGISTER(MaskOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "MaskOp", "Tensor mask operation using relational comparator") + .def(py::init, DataType>()); + })); + +PYBIND_REGISTER(OneHotOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") + .def(py::init()); + })); + +PYBIND_REGISTER(PadEndOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "PadEndOp", "Tensor operation to pad end of tensor with a pad value.") + .def(py::init>()); + })); + +PYBIND_REGISTER(SliceOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "SliceOp", + "Tensor slice operation.") + .def(py::init()) + .def(py::init([](const py::list &py_list) { + std::vector c_list; + for (auto l : py_list) { + if (!l.is_none()) { + c_list.push_back(py::reinterpret_borrow(l)); + } + } + return std::make_shared(c_list); + })) + .def(py::init([](const py::tuple &py_slice) { + if (py_slice.size() != 3) { + THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Wrong slice object")); + } + Slice c_slice; + if (!py_slice[0].is_none() && !py_slice[1].is_none() && !py_slice[2].is_none()) { + c_slice = Slice(py::reinterpret_borrow(py_slice[0]), + py::reinterpret_borrow(py_slice[1]), + py::reinterpret_borrow(py_slice[2])); + } else if (py_slice[0].is_none() && py_slice[2].is_none()) { + c_slice = Slice(py::reinterpret_borrow(py_slice[1])); + } else if (!py_slice[0].is_none() && !py_slice[1].is_none()) { + c_slice = Slice(py::reinterpret_borrow(py_slice[0]), + py::reinterpret_borrow(py_slice[1])); + } + + if (!c_slice.valid()) { + THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Wrong slice object")); + } + return std::make_shared(c_slice); + })); + })); + +PYBIND_REGISTER(ToFloat16Op, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "ToFloat16Op", py::dynamic_attr(), + "Tensor operator to type cast float32 data to a float16 type.") + .def(py::init<>()); + })); + +PYBIND_REGISTER(TypeCastOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "TypeCastOp", "Tensor operator to type cast data to a specified type.") + .def(py::init(), py::arg("data_type")) + .def(py::init(), py::arg("data_type")); + })); + +PYBIND_REGISTER(RelationalOp, 0, ([](const py::module *m) { + (void)py::enum_(*m, "RelationalOp", py::arithmetic()) + .value("EQ", RelationalOp::kEqual) + .value("NE", RelationalOp::kNotEqual) + .value("LT", RelationalOp::kLess) + .value("LE", RelationalOp::kLessEqual) + .value("GT", RelationalOp::kGreater) + .value("GE", RelationalOp::kGreaterEqual) + .export_values(); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/image/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/image/bindings.cc new file mode 100644 index 0000000000..b2873cd5a4 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/image/bindings.cc @@ -0,0 +1,346 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/kernels/py_func_op.h" +#include "minddata/dataset/kernels/image/auto_contrast_op.h" +#include "minddata/dataset/kernels/image/bounding_box_augment_op.h" +#include "minddata/dataset/kernels/image/center_crop_op.h" +#include "minddata/dataset/kernels/image/cut_out_op.h" +#include "minddata/dataset/kernels/image/decode_op.h" +#include "minddata/dataset/kernels/image/equalize_op.h" +#include "minddata/dataset/kernels/image/hwc_to_chw_op.h" +#include "minddata/dataset/kernels/image/image_utils.h" +#include "minddata/dataset/kernels/image/invert_op.h" +#include "minddata/dataset/kernels/image/normalize_op.h" +#include "minddata/dataset/kernels/image/pad_op.h" +#include "minddata/dataset/kernels/image/random_color_adjust_op.h" +#include "minddata/dataset/kernels/image/random_crop_and_resize_op.h" +#include "minddata/dataset/kernels/image/random_crop_and_resize_with_bbox_op.h" +#include "minddata/dataset/kernels/image/random_crop_decode_resize_op.h" +#include "minddata/dataset/kernels/image/random_crop_op.h" +#include "minddata/dataset/kernels/image/random_crop_with_bbox_op.h" +#include "minddata/dataset/kernels/image/random_horizontal_flip_op.h" +#include "minddata/dataset/kernels/image/random_horizontal_flip_with_bbox_op.h" +#include "minddata/dataset/kernels/image/random_resize_op.h" +#include "minddata/dataset/kernels/image/random_resize_with_bbox_op.h" +#include "minddata/dataset/kernels/image/random_rotation_op.h" +#include "minddata/dataset/kernels/image/random_select_subpolicy_op.h" +#include "minddata/dataset/kernels/image/random_vertical_flip_op.h" +#include "minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h" +#include "minddata/dataset/kernels/image/rescale_op.h" +#include "minddata/dataset/kernels/image/resize_bilinear_op.h" +#include "minddata/dataset/kernels/image/resize_op.h" +#include "minddata/dataset/kernels/image/resize_with_bbox_op.h" +#include "minddata/dataset/kernels/image/uniform_aug_op.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(AutoContrastOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "AutoContrastOp", "Tensor operation to apply autocontrast on an image.") + .def(py::init>(), py::arg("cutoff") = AutoContrastOp::kCutOff, + py::arg("ignore") = AutoContrastOp::kIgnore); + })); + +PYBIND_REGISTER(NormalizeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "NormalizeOp", "Tensor operation to normalize an image. Takes mean and std.") + .def(py::init(), py::arg("meanR"), py::arg("meanG"), + py::arg("meanB"), py::arg("stdR"), py::arg("stdG"), py::arg("stdB")); + })); + +PYBIND_REGISTER(EqualizeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "EqualizeOp", "Tensor operation to apply histogram equalization on images.") + .def(py::init<>()); + })); + +PYBIND_REGISTER(InvertOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "InvertOp", "Tensor operation to apply invert on RGB images.") + .def(py::init<>()); + })); + +PYBIND_REGISTER(RescaleOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RescaleOp", "Tensor operation to rescale an image. Takes scale and shift.") + .def(py::init(), py::arg("rescale"), py::arg("shift")); + })); + +PYBIND_REGISTER(CenterCropOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "CenterCropOp", + "Tensor operation to crop and image in the middle. Takes height and width (optional)") + .def(py::init(), py::arg("height"), py::arg("width") = CenterCropOp::kDefWidth); + })); + +PYBIND_REGISTER(ResizeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "ResizeOp", "Tensor operation to resize an image. Takes height, width and mode") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = ResizeOp::kDefWidth, + py::arg("interpolation") = ResizeOp::kDefInterpolation); + })); + +PYBIND_REGISTER(ResizeWithBBoxOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "ResizeWithBBoxOp", "Tensor operation to resize an image. Takes height, width and mode.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = ResizeWithBBoxOp::kDefWidth, + py::arg("interpolation") = ResizeWithBBoxOp::kDefInterpolation); + })); + +PYBIND_REGISTER( + RandomResizeWithBBoxOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomResizeWithBBoxOp", + "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = RandomResizeWithBBoxOp::kDefTargetWidth); + })); +PYBIND_REGISTER(UniformAugOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "UniformAugOp", "Tensor operation to apply random augmentation(s).") + .def(py::init>, int32_t>(), py::arg("transforms"), + py::arg("NumOps") = UniformAugOp::kDefNumOps); + })); +PYBIND_REGISTER(BoundingBoxAugmentOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "BoundingBoxAugmentOp", + "Tensor operation to apply a transformation on a random choice of bounding boxes.") + .def(py::init, float>(), py::arg("transform"), + py::arg("ratio") = BoundingBoxAugmentOp::kDefRatio); + })); +PYBIND_REGISTER(ResizeBilinearOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "ResizeBilinearOp", + "Tensor operation to resize an image using " + "Bilinear mode. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = ResizeBilinearOp::kDefWidth); + })); + +PYBIND_REGISTER(DecodeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "DecodeOp", "Tensor operation to decode a jpg image") + .def(py::init<>()) + .def(py::init(), py::arg("rgb_format") = DecodeOp::kDefRgbFormat); + })); + +PYBIND_REGISTER(RandomHorizontalFlipOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomHorizontalFlipOp", "Tensor operation to randomly flip an image horizontally.") + .def(py::init(), py::arg("probability") = RandomHorizontalFlipOp::kDefProbability); + })); + +PYBIND_REGISTER( + RandomHorizontalFlipWithBBoxOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomHorizontalFlipWithBBoxOp", + "Tensor operation to randomly flip an image horizontally, while flipping bounding boxes.") + .def(py::init(), py::arg("probability") = RandomHorizontalFlipWithBBoxOp::kDefProbability); + })); +PYBIND_REGISTER(RandomVerticalFlipOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomVerticalFlipOp", "Tensor operation to randomly flip an image vertically.") + .def(py::init(), py::arg("probability") = RandomVerticalFlipOp::kDefProbability); + })); +PYBIND_REGISTER(RandomVerticalFlipWithBBoxOp, 1, ([](const py::module *m) { + (void) + py::class_>( + *m, "RandomVerticalFlipWithBBoxOp", + "Tensor operation to randomly flip an image vertically" + " and adjust bounding boxes.") + .def(py::init(), py::arg("probability") = RandomVerticalFlipWithBBoxOp::kDefProbability); + })); +PYBIND_REGISTER( + RandomCropOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "RandomCropOp", + "Gives random crop of specified size " + "Takes crop size") + .def( + py::init(), + py::arg("cropHeight"), py::arg("cropWidth"), py::arg("padTop") = RandomCropOp::kDefPadTop, + py::arg("padBottom") = RandomCropOp::kDefPadBottom, py::arg("padLeft") = RandomCropOp::kDefPadLeft, + py::arg("padRight") = RandomCropOp::kDefPadRight, py::arg("borderType") = RandomCropOp::kDefBorderType, + py::arg("padIfNeeded") = RandomCropOp::kDefPadIfNeeded, py::arg("fillR") = RandomCropOp::kDefFillR, + py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); + })); +PYBIND_REGISTER( + HwcToChwOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "ChannelSwapOp").def(py::init<>()); + })); +PYBIND_REGISTER( + RandomCropWithBBoxOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomCropWithBBoxOp", + "Gives random crop of given " + "size + adjusts bboxes " + "Takes crop size") + .def( + py::init(), + py::arg("cropHeight"), py::arg("cropWidth"), py::arg("padTop") = RandomCropWithBBoxOp::kDefPadTop, + py::arg("padBottom") = RandomCropWithBBoxOp::kDefPadBottom, + py::arg("padLeft") = RandomCropWithBBoxOp::kDefPadLeft, + py::arg("padRight") = RandomCropWithBBoxOp::kDefPadRight, + py::arg("borderType") = RandomCropWithBBoxOp::kDefBorderType, + py::arg("padIfNeeded") = RandomCropWithBBoxOp::kDefPadIfNeeded, + py::arg("fillR") = RandomCropWithBBoxOp::kDefFillR, py::arg("fillG") = RandomCropWithBBoxOp::kDefFillG, + py::arg("fillB") = RandomCropWithBBoxOp::kDefFillB); + })); +PYBIND_REGISTER(CutOutOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "CutOutOp", + "Tensor operation to randomly erase a portion of the image. Takes height and width.") + .def(py::init(), py::arg("boxHeight"), + py::arg("boxWidth"), py::arg("numPatches"), py::arg("randomColor") = CutOutOp::kDefRandomColor, + py::arg("fillR") = CutOutOp::kDefFillR, py::arg("fillG") = CutOutOp::kDefFillG, + py::arg("fillB") = CutOutOp::kDefFillB); + })); +PYBIND_REGISTER(PadOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "PadOp", + "Pads image with specified color, default black, " + "Takes amount to pad for top, bottom, left, right of image, boarder type and color") + .def(py::init(), + py::arg("padTop"), py::arg("padBottom"), py::arg("padLeft"), py::arg("padRight"), + py::arg("borderTypes") = PadOp::kDefBorderType, py::arg("fillR") = PadOp::kDefFillR, + py::arg("fillG") = PadOp::kDefFillG, py::arg("fillB") = PadOp::kDefFillB); + })); + +PYBIND_REGISTER(RandomCropDecodeResizeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomCropDecodeResizeOp", "equivalent to RandomCropAndResize but crops before decoding") + .def(py::init(), + py::arg("targetHeight"), py::arg("targetWidth"), + py::arg("scaleLb") = RandomCropDecodeResizeOp::kDefScaleLb, + py::arg("scaleUb") = RandomCropDecodeResizeOp::kDefScaleUb, + py::arg("aspectLb") = RandomCropDecodeResizeOp::kDefAspectLb, + py::arg("aspectUb") = RandomCropDecodeResizeOp::kDefAspectUb, + py::arg("interpolation") = RandomCropDecodeResizeOp::kDefInterpolation, + py::arg("maxIter") = RandomCropDecodeResizeOp::kDefMaxIter); + })); + +PYBIND_REGISTER( + RandomResizeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomResizeOp", + "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = RandomResizeOp::kDefTargetWidth); + })); + +PYBIND_REGISTER(RandomColorAdjustOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomColorAdjustOp", + "Tensor operation to adjust an image's color randomly." + "Takes range for brightness, contrast, saturation, hue and") + .def(py::init(), + py::arg("bright_factor_start"), py::arg("bright_factor_end"), py::arg("contrast_factor_start"), + py::arg("contrast_factor_end"), py::arg("saturation_factor_start"), + py::arg("saturation_factor_end"), py::arg("hue_factor_start"), py::arg("hue_factor_end")); + })); + +PYBIND_REGISTER(RandomCropAndResizeWithBBoxOp, 1, ([](const py::module *m) { + (void) + py::class_>( + *m, "RandomCropAndResizeWithBBoxOp", + "Tensor operation to randomly crop an image (with BBoxes) and resize to a given size." + "Takes output height and width and" + "optional parameters for lower and upper bound for aspect ratio (h/w) and scale," + "interpolation mode, and max attempts to crop") + .def(py::init(), + py::arg("targetHeight"), py::arg("targetWidth"), + py::arg("scaleLb") = RandomCropAndResizeWithBBoxOp::kDefScaleLb, + py::arg("scaleUb") = RandomCropAndResizeWithBBoxOp::kDefScaleUb, + py::arg("aspectLb") = RandomCropAndResizeWithBBoxOp::kDefAspectLb, + py::arg("aspectUb") = RandomCropAndResizeWithBBoxOp::kDefAspectUb, + py::arg("interpolation") = RandomCropAndResizeWithBBoxOp::kDefInterpolation, + py::arg("maxIter") = RandomCropAndResizeWithBBoxOp::kDefMaxIter); + })); + +PYBIND_REGISTER(RandomCropAndResizeOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomCropAndResizeOp", + "Tensor operation to randomly crop an image and resize to a given size." + "Takes output height and width and" + "optional parameters for lower and upper bound for aspect ratio (h/w) and scale," + "interpolation mode, and max attempts to crop") + .def(py::init(), + py::arg("targetHeight"), py::arg("targetWidth"), + py::arg("scaleLb") = RandomCropAndResizeOp::kDefScaleLb, + py::arg("scaleUb") = RandomCropAndResizeOp::kDefScaleUb, + py::arg("aspectLb") = RandomCropAndResizeOp::kDefAspectLb, + py::arg("aspectUb") = RandomCropAndResizeOp::kDefAspectUb, + py::arg("interpolation") = RandomCropAndResizeOp::kDefInterpolation, + py::arg("maxIter") = RandomCropAndResizeOp::kDefMaxIter); + })); + +PYBIND_REGISTER(RandomRotationOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomRotationOp", + "Tensor operation to apply RandomRotation." + "Takes a range for degrees and " + "optional parameters for rotation center and image expand") + .def( + py::init(), + py::arg("startDegree"), py::arg("endDegree"), py::arg("centerX") = RandomRotationOp::kDefCenterX, + py::arg("centerY") = RandomRotationOp::kDefCenterY, + py::arg("interpolation") = RandomRotationOp::kDefInterpolation, + py::arg("expand") = RandomRotationOp::kDefExpand, py::arg("fillR") = RandomRotationOp::kDefFillR, + py::arg("fillG") = RandomRotationOp::kDefFillG, py::arg("fillB") = RandomRotationOp::kDefFillB); + })); + +PYBIND_REGISTER(RandomSelectSubpolicyOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RandomSelectSubpolicyOp") + .def(py::init([](const py::list &py_policy) { + std::vector cpp_policy; + for (auto &py_sub : py_policy) { + cpp_policy.push_back({}); + for (auto handle : py_sub.cast()) { + py::tuple tp = handle.cast(); + if (tp.is_none() || tp.size() != 2) { + THROW_IF_ERROR( + Status(StatusCode::kUnexpectedError, "Each tuple in subpolicy should be (op, prob).")); + } + std::shared_ptr t_op; + if (py::isinstance(tp[0])) { + t_op = (tp[0]).cast>(); + } else if (py::isinstance(tp[0])) { + t_op = std::make_shared((tp[0]).cast()); + } else { + THROW_IF_ERROR( + Status(StatusCode::kUnexpectedError, "op is neither a tensorOp nor a pyfunc.")); + } + double prob = (tp[1]).cast(); + if (prob < 0 || prob > 1) { + THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "prob needs to be with [0,1].")); + } + cpp_policy.back().emplace_back(std::make_pair(t_op, prob)); + } + } + return std::make_shared(cpp_policy); + })); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc new file mode 100644 index 0000000000..244b3402ab --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/text/vocab.h" +#include "minddata/dataset/text/sentence_piece_vocab.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(Vocab, 0, ([](const py::module *m) { + (void)py::class_>(*m, "Vocab") + .def(py::init<>()) + .def_static("from_list", + [](const py::list &words, const py::list &special_tokens, bool special_first) { + std::shared_ptr v; + THROW_IF_ERROR(Vocab::BuildFromPyList(words, special_tokens, special_first, &v)); + return v; + }) + .def_static( + "from_file", + [](const std::string &path, const std::string &dlm, int32_t vocab_size, + const py::list &special_tokens, bool special_first) { + std::shared_ptr v; + THROW_IF_ERROR(Vocab::BuildFromFile(path, dlm, vocab_size, special_tokens, special_first, &v)); + return v; + }) + .def_static("from_dict", [](const py::dict &words) { + std::shared_ptr v; + THROW_IF_ERROR(Vocab::BuildFromPyDict(words, &v)); + return v; + }); + })); + +PYBIND_REGISTER(SentencePieceVocab, 0, ([](const py::module *m) { + (void)py::class_>(*m, "SentencePieceVocab") + .def(py::init<>()) + .def_static("from_file", + [](const py::list &paths, const int vocab_size, const float character_coverage, + const SentencePieceModel model_type, const py::dict ¶ms) { + std::shared_ptr v; + std::vector path_list; + for (auto path : paths) { + path_list.emplace_back(py::str(path)); + } + std::unordered_map param_map; + for (auto param : params) { + std::string key = py::reinterpret_borrow(param.first); + if (key == "input" || key == "vocab_size" || key == "model_prefix" || + key == "character_coverage" || key == "model_type") { + continue; + } + param_map[key] = py::reinterpret_borrow(param.second); + } + THROW_IF_ERROR(SentencePieceVocab::BuildFromFile( + path_list, vocab_size, character_coverage, model_type, param_map, &v)); + return v; + }) + .def_static("save_model", [](const std::shared_ptr *vocab, std::string path, + std::string filename) { + THROW_IF_ERROR(SentencePieceVocab::SaveModel(vocab, path, filename)); + }); + })); + +PYBIND_REGISTER(SentencePieceModel, 0, ([](const py::module *m) { + (void)py::enum_(*m, "SentencePieceModel", py::arithmetic()) + .value("DE_SENTENCE_PIECE_UNIGRAM", SentencePieceModel::kUnigram) + .value("DE_SENTENCE_PIECE_BPE", SentencePieceModel::kBpe) + .value("DE_SENTENCE_PIECE_CHAR", SentencePieceModel::kChar) + .value("DE_SENTENCE_PIECE_WORD", SentencePieceModel::kWord) + .export_values(); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/bindings.cc new file mode 100644 index 0000000000..5949b0f358 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/bindings.cc @@ -0,0 +1,244 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" +#include "minddata/dataset/api/python/pybind_register.h" + +#include "minddata/dataset/text/kernels/jieba_tokenizer_op.h" +#include "minddata/dataset/text/kernels/lookup_op.h" +#include "minddata/dataset/text/kernels/ngram_op.h" +#include "minddata/dataset/text/kernels/sliding_window_op.h" +#include "minddata/dataset/text/kernels/to_number_op.h" +#include "minddata/dataset/text/kernels/unicode_char_tokenizer_op.h" +#include "minddata/dataset/text/kernels/wordpiece_tokenizer_op.h" +#include "minddata/dataset/text/kernels/sentence_piece_tokenizer_op.h" +#include "minddata/dataset/text/kernels/truncate_sequence_pair_op.h" + +#ifdef ENABLE_ICU4C +#include "minddata/dataset/text/kernels/basic_tokenizer_op.h" +#include "minddata/dataset/text/kernels/bert_tokenizer_op.h" +#include "minddata/dataset/text/kernels/case_fold_op.h" +#include "minddata/dataset/text/kernels/normalize_utf8_op.h" +#include "minddata/dataset/text/kernels/regex_replace_op.h" +#include "minddata/dataset/text/kernels/regex_tokenizer_op.h" +#include "minddata/dataset/text/kernels/unicode_script_tokenizer_op.h" +#include "minddata/dataset/text/kernels/whitespace_tokenizer_op.h" +#endif + +namespace mindspore { +namespace dataset { + +#ifdef ENABLE_ICU4C + +PYBIND_REGISTER(BasicTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "BasicTokenizerOp", "Tokenize a scalar tensor of UTF-8 string by specific rules.") + .def(py::init(), + py::arg("lower_case") = BasicTokenizerOp::kDefLowerCase, + py::arg("keep_whitespace") = BasicTokenizerOp::kDefKeepWhitespace, + py::arg("normalization_form") = BasicTokenizerOp::kDefNormalizationForm, + py::arg("preserve_unused_token") = BasicTokenizerOp::kDefPreserveUnusedToken, + py::arg("with_offsets") = BasicTokenizerOp::kDefWithOffsets); + })); + +PYBIND_REGISTER(WhitespaceTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "WhitespaceTokenizerOp", "Tokenize a scalar tensor of UTF-8 string on ICU defined whitespaces.") + .def(py::init(), py::arg(" with_offsets ") = WhitespaceTokenizerOp::kDefWithOffsets); + })); + +PYBIND_REGISTER(UnicodeScriptTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "UnicodeScriptTokenizerOp", + "Tokenize a scalar tensor of UTF-8 string on Unicode script boundaries.") + .def(py::init<>()) + .def(py::init(), + py::arg("keep_whitespace") = UnicodeScriptTokenizerOp::kDefKeepWhitespace, + py::arg("with_offsets") = UnicodeScriptTokenizerOp::kDefWithOffsets); + })); + +PYBIND_REGISTER(CaseFoldOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "CaseFoldOp", "Apply case fold operation on utf-8 string tensor") + .def(py::init<>()); + })); + +PYBIND_REGISTER(NormalizeUTF8Op, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "NormalizeUTF8Op", "Apply normalize operation on utf-8 string tensor.") + .def(py::init<>()) + .def(py::init(), py::arg("normalize_form") = NormalizeUTF8Op::kDefNormalizeForm); + })); + +PYBIND_REGISTER(RegexReplaceOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RegexReplaceOp", + "Replace utf-8 string tensor with 'replace' according to regular expression 'pattern'.") + .def(py::init(), py::arg("pattern"), + py::arg("replace"), py::arg("replace_all")); + })); + +PYBIND_REGISTER(RegexTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "RegexTokenizerOp", "Tokenize a scalar tensor of UTF-8 string by regex expression pattern.") + .def(py::init(), py::arg("delim_pattern"), + py::arg("keep_delim_pattern"), py::arg("with_offsets") = RegexTokenizerOp::kDefWithOffsets); + })); +PYBIND_REGISTER(BertTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "BertTokenizerOp", "Tokenizer used for Bert text process.") + .def(py::init &, const std::string &, const int &, const std::string &, + const bool &, const bool &, const NormalizeForm &, const bool &, const bool &>(), + py::arg("vocab"), + py::arg("suffix_indicator") = std::string(WordpieceTokenizerOp::kDefSuffixIndicator), + py::arg("max_bytes_per_token") = WordpieceTokenizerOp::kDefMaxBytesPerToken, + py::arg("unknown_token") = std::string(WordpieceTokenizerOp::kDefUnknownToken), + py::arg("lower_case") = BasicTokenizerOp::kDefLowerCase, + py::arg("keep_whitespace") = BasicTokenizerOp::kDefKeepWhitespace, + py::arg("normalization_form") = BasicTokenizerOp::kDefNormalizationForm, + py::arg("preserve_unused_token") = BasicTokenizerOp::kDefPreserveUnusedToken, + py::arg("with_offsets") = WordpieceTokenizerOp::kDefWithOffsets); + })); + +PYBIND_REGISTER(NormalizeForm, 0, ([](const py::module *m) { + (void)py::enum_(*m, "NormalizeForm", py::arithmetic()) + .value("DE_NORMALIZE_NONE", NormalizeForm::kNone) + .value("DE_NORMALIZE_NFC", NormalizeForm::kNfc) + .value("DE_NORMALIZE_NFKC", NormalizeForm::kNfkc) + .value("DE_NORMALIZE_NFD", NormalizeForm::kNfd) + .value("DE_NORMALIZE_NFKD", NormalizeForm::kNfkd) + .export_values(); + })); + +#endif + +PYBIND_REGISTER(JiebaTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "JiebaTokenizerOp", "") + .def(py::init(), + py::arg("hmm_path"), py::arg("mp_path"), py::arg("mode") = JiebaMode::kMix, + py::arg("with_offsets") = JiebaTokenizerOp::kDefWithOffsets) + .def("add_word", [](JiebaTokenizerOp &self, const std::string word, int freq) { + THROW_IF_ERROR(self.AddWord(word, freq)); + }); + })); + +PYBIND_REGISTER(UnicodeCharTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "UnicodeCharTokenizerOp", "Tokenize a scalar tensor of UTF-8 string to Unicode characters.") + .def(py::init(), py::arg("with_offsets") = UnicodeCharTokenizerOp::kDefWithOffsets); + })); + +PYBIND_REGISTER(LookupOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "LookupOp", "Tensor operation to LookUp each word.") + .def(py::init([](std::shared_ptr vocab, const py::object &py_word) { + if (vocab == nullptr) { + THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "vocab object type is incorrect or null.")); + } + if (py_word.is_none()) { + return std::make_shared(vocab, Vocab::kNoTokenExists); + } + std::string word = py::reinterpret_borrow(py_word); + WordIdType default_id = vocab->Lookup(word); + if (default_id == Vocab::kNoTokenExists) { + THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, + "default unknown token: " + word + " doesn't exist in vocab.")); + } + return std::make_shared(vocab, default_id); + })); + })); + +PYBIND_REGISTER(NgramOp, 1, ([](const py::module *m) { + (void)py::class_>(*m, "NgramOp", + "TensorOp performs ngram mapping.") + .def(py::init &, int32_t, int32_t, const std::string &, + const std::string &, const std::string &>(), + py::arg("ngrams"), py::arg("l_pad_len"), py::arg("r_pad_len"), py::arg("l_pad_token"), + py::arg("r_pad_token"), py::arg("separator")); + })); + +PYBIND_REGISTER( + WordpieceTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "WordpieceTokenizerOp", "Tokenize scalar token or 1-D tokens to subword tokens.") + .def( + py::init &, const std::string &, const int &, const std::string &, const bool &>(), + py::arg("vocab"), py::arg("suffix_indicator") = std::string(WordpieceTokenizerOp::kDefSuffixIndicator), + py::arg("max_bytes_per_token") = WordpieceTokenizerOp::kDefMaxBytesPerToken, + py::arg("unknown_token") = std::string(WordpieceTokenizerOp::kDefUnknownToken), + py::arg("with_offsets") = WordpieceTokenizerOp::kDefWithOffsets); + })); + +PYBIND_REGISTER(SlidingWindowOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "SlidingWindowOp", "TensorOp to apply sliding window to a 1-D Tensor.") + .def(py::init(), py::arg("width"), py::arg("axis")); + })); + +PYBIND_REGISTER( + SentencePieceTokenizerOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "SentencePieceTokenizerOp", "Tokenize scalar token or 1-D tokens to tokens by sentence piece.") + .def( + py::init &, const SPieceTokenizerLoadType, const SPieceTokenizerOutType>(), + py::arg("vocab"), py::arg("load_type") = SPieceTokenizerLoadType::kModel, + py::arg("out_type") = SPieceTokenizerOutType::kString) + .def(py::init(), + py::arg("model_path"), py::arg("model_filename"), py::arg("load_type") = SPieceTokenizerLoadType::kFile, + py::arg("out_type") = SPieceTokenizerOutType::kString); + })); + +PYBIND_REGISTER(ToNumberOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "ToNumberOp", "TensorOp to convert strings to numbers.") + .def(py::init(), py::arg("data_type")) + .def(py::init(), py::arg("data_type")); + })); + +PYBIND_REGISTER(TruncateSequencePairOp, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "TruncateSequencePairOp", "Tensor operation to truncate two tensors to a max_length") + .def(py::init()); + })); + +PYBIND_REGISTER(JiebaMode, 0, ([](const py::module *m) { + (void)py::enum_(*m, "JiebaMode", py::arithmetic()) + .value("DE_JIEBA_MIX", JiebaMode::kMix) + .value("DE_JIEBA_MP", JiebaMode::kMp) + .value("DE_JIEBA_HMM", JiebaMode::kHmm) + .export_values(); + })); + +PYBIND_REGISTER(SPieceTokenizerOutType, 0, ([](const py::module *m) { + (void)py::enum_(*m, "SPieceTokenizerOutType", py::arithmetic()) + .value("DE_SPIECE_TOKENIZER_OUTTYPE_KString", SPieceTokenizerOutType::kString) + .value("DE_SPIECE_TOKENIZER_OUTTYPE_KINT", SPieceTokenizerOutType::kInt) + .export_values(); + })); + +PYBIND_REGISTER(SPieceTokenizerLoadType, 0, ([](const py::module *m) { + (void)py::enum_(*m, "SPieceTokenizerLoadType", py::arithmetic()) + .value("DE_SPIECE_TOKENIZER_LOAD_KFILE", SPieceTokenizerLoadType::kFile) + .value("DE_SPIECE_TOKENIZER_LOAD_KMODEL", SPieceTokenizerLoadType::kModel) + .export_values(); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/mindrecord/include/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/mindrecord/include/bindings.cc new file mode 100644 index 0000000000..5c785fcd37 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/mindrecord/include/bindings.cc @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pybind11/pybind11.h" +#include "pybind11/stl_bind.h" + +#include "minddata/dataset/api/python/pybind_register.h" + +#include "minddata/dataset/util/random.h" +#include "minddata/mindrecord/include/shard_distributed_sample.h" +#include "minddata/mindrecord/include/shard_operator.h" +#include "minddata/mindrecord/include/shard_pk_sample.h" +#include "minddata/mindrecord/include/shard_sample.h" +#include "minddata/mindrecord/include/shard_sequential_sample.h" +#include "minddata/mindrecord/include/shard_shuffle.h" + +namespace mindspore { +namespace dataset { + +PYBIND_REGISTER(ShardOperator, 0, ([](const py::module *m) { + (void)py::class_>( + *m, "ShardOperator") + .def("add_child", + [](std::shared_ptr self, + std::shared_ptr child) { self->SetChildOp(child); }); + })); + +PYBIND_REGISTER(ShardDistributedSample, 1, ([](const py::module *m) { + (void)py::class_>(*m, + "MindrecordDistributedSampler") + .def(py::init()); + })); + +PYBIND_REGISTER( + ShardPkSample, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "MindrecordPkSampler") + .def(py::init([](int64_t kVal, std::string kColumn, bool shuffle) { + if (shuffle == true) { + return std::make_shared(kColumn, kVal, std::numeric_limits::max(), + GetSeed()); + } else { + return std::make_shared(kColumn, kVal); + } + })); + })); + +PYBIND_REGISTER( + ShardSample, 0, ([](const py::module *m) { + (void)py::class_>( + *m, "MindrecordSubsetRandomSampler") + .def(py::init, uint32_t>(), py::arg("indices"), py::arg("seed") = GetSeed()); + })); + +PYBIND_REGISTER(ShardSequentialSample, 0, ([](const py::module *m) { + (void)py::class_>(*m, + "MindrecordSequentialSampler") + .def(py::init([](int num_samples, int start_index) { + return std::make_shared(num_samples, start_index); + })); + })); + +PYBIND_REGISTER( + ShardShuffle, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "MindrecordRandomSampler") + .def(py::init([](int64_t num_samples, bool replacement, bool reshuffle_each_epoch) { + return std::make_shared(GetSeed(), num_samples, replacement, reshuffle_each_epoch); + })); + })); + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/de_pipeline.cc b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc similarity index 99% rename from mindspore/ccsrc/minddata/dataset/api/de_pipeline.cc rename to mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc index ce4137c1d0..408ed3b270 100644 --- a/mindspore/ccsrc/minddata/dataset/api/de_pipeline.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "minddata/dataset/api/de_pipeline.h" +#include "minddata/dataset/api/python/de_pipeline.h" #include #include diff --git a/mindspore/ccsrc/minddata/dataset/api/de_pipeline.h b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.h similarity index 100% rename from mindspore/ccsrc/minddata/dataset/api/de_pipeline.h rename to mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.h diff --git a/mindspore/ccsrc/minddata/dataset/api/python/pybind_register.cc b/mindspore/ccsrc/minddata/dataset/api/python/pybind_register.cc new file mode 100644 index 0000000000..11a520be5e --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/pybind_register.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/api/python/pybind_register.h" + +namespace mindspore { +namespace dataset { + +PybindDefinedFunctionRegister &PybindDefinedFunctionRegister::GetSingleton() { + static PybindDefinedFunctionRegister instance; + return instance; +} + +// This is where we externalize the C logic as python modules +PYBIND11_MODULE(_c_dataengine, m) { + m.doc() = "pybind11 for _c_dataengine"; + + auto all_fns = mindspore::dataset::PybindDefinedFunctionRegister::AllFunctions(); + + for (auto &item : all_fns) { + for (auto &func : item.second) { + func.second(&m); + } + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/api/python/pybind_register.h b/mindspore/ccsrc/minddata/dataset/api/python/pybind_register.h new file mode 100644 index 0000000000..8717a2844c --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/api/python/pybind_register.h @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef API_PYBIND_API_H_ +#define API_PYBIND_API_H_ + +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +namespace py = pybind11; +namespace mindspore { + +namespace dataset { +#define THROW_IF_ERROR(s) \ + do { \ + Status rc = std::move(s); \ + if (rc.IsError()) throw std::runtime_error(rc.ToString()); \ + } while (false) + +using PybindDefineFunc = std::function; + +class PybindDefinedFunctionRegister { + public: + static void Register(const std::string &name, const uint8_t &priority, const PybindDefineFunc &fn) { + return GetSingleton().RegisterFn(name, priority, fn); + } + + PybindDefinedFunctionRegister(const PybindDefinedFunctionRegister &) = delete; + + PybindDefinedFunctionRegister &operator=(const PybindDefinedFunctionRegister &) = delete; + + static std::map> &AllFunctions() { + return GetSingleton().module_fns_; + } + std::map> module_fns_; + + protected: + PybindDefinedFunctionRegister() = default; + + virtual ~PybindDefinedFunctionRegister() = default; + + static PybindDefinedFunctionRegister &GetSingleton(); + + void RegisterFn(const std::string &name, const uint8_t &priority, const PybindDefineFunc &fn) { + module_fns_[priority][name] = fn; + } +}; + +class PybindDefineRegisterer { + public: + PybindDefineRegisterer(const std::string &name, const uint8_t &priority, const PybindDefineFunc &fn) { + PybindDefinedFunctionRegister::Register(name, priority, fn); + } + ~PybindDefineRegisterer() = default; +}; + +#ifdef ENABLE_PYTHON +#define PYBIND_REGISTER(name, priority, define) PybindDefineRegisterer g_pybind_define_f_##name(#name, priority, define) +#endif +} // namespace dataset +} // namespace mindspore +#endif // API_PYBIND_API_H_ diff --git a/mindspore/ccsrc/minddata/dataset/api/python_bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python_bindings.cc deleted file mode 100644 index c6a9eb0aea..0000000000 --- a/mindspore/ccsrc/minddata/dataset/api/python_bindings.cc +++ /dev/null @@ -1,1138 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "minddata/dataset/api/de_pipeline.h" -#include "minddata/dataset/engine/cache/cache_client.h" -#include "minddata/dataset/engine/datasetops/source/cifar_op.h" -#include "minddata/dataset/engine/datasetops/source/clue_op.h" -#include "minddata/dataset/engine/datasetops/source/csv_op.h" -#include "minddata/dataset/engine/datasetops/source/coco_op.h" -#include "minddata/dataset/engine/datasetops/source/image_folder_op.h" -#include "minddata/dataset/engine/datasetops/source/io_block.h" -#include "minddata/dataset/engine/datasetops/source/manifest_op.h" -#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h" -#include "minddata/dataset/engine/datasetops/source/mnist_op.h" -#include "minddata/dataset/engine/datasetops/source/random_data_op.h" -#include "minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.h" -#include "minddata/dataset/engine/datasetops/source/sampler/pk_sampler.h" -#include "minddata/dataset/engine/datasetops/source/sampler/python_sampler.h" -#include "minddata/dataset/engine/datasetops/source/sampler/random_sampler.h" -#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" -#include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h" -#include "minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" -#include "minddata/dataset/engine/datasetops/source/text_file_op.h" -#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" -#include "minddata/dataset/engine/datasetops/source/voc_op.h" -#include "minddata/dataset/engine/gnn/graph.h" -#include "minddata/dataset/engine/jagged_connector.h" -#include "minddata/dataset/kernels/compose_op.h" -#include "minddata/dataset/kernels/data/concatenate_op.h" -#include "minddata/dataset/kernels/data/duplicate_op.h" -#include "minddata/dataset/kernels/data/fill_op.h" -#include "minddata/dataset/kernels/data/mask_op.h" -#include "minddata/dataset/kernels/data/one_hot_op.h" -#include "minddata/dataset/kernels/data/pad_end_op.h" -#include "minddata/dataset/kernels/data/slice_op.h" -#include "minddata/dataset/kernels/data/to_float16_op.h" -#include "minddata/dataset/kernels/data/type_cast_op.h" -#include "minddata/dataset/kernels/image/auto_contrast_op.h" -#include "minddata/dataset/kernels/image/bounding_box_augment_op.h" -#include "minddata/dataset/kernels/image/center_crop_op.h" -#include "minddata/dataset/kernels/image/cut_out_op.h" -#include "minddata/dataset/kernels/image/decode_op.h" -#include "minddata/dataset/kernels/image/equalize_op.h" -#include "minddata/dataset/kernels/image/hwc_to_chw_op.h" -#include "minddata/dataset/kernels/image/image_utils.h" -#include "minddata/dataset/kernels/image/invert_op.h" -#include "minddata/dataset/kernels/image/normalize_op.h" -#include "minddata/dataset/kernels/image/pad_op.h" -#include "minddata/dataset/kernels/image/random_color_adjust_op.h" -#include "minddata/dataset/kernels/image/random_crop_and_resize_op.h" -#include "minddata/dataset/kernels/image/random_crop_and_resize_with_bbox_op.h" -#include "minddata/dataset/kernels/image/random_crop_decode_resize_op.h" -#include "minddata/dataset/kernels/image/random_crop_op.h" -#include "minddata/dataset/kernels/image/random_crop_with_bbox_op.h" -#include "minddata/dataset/kernels/image/random_horizontal_flip_op.h" -#include "minddata/dataset/kernels/image/random_horizontal_flip_with_bbox_op.h" -#include "minddata/dataset/kernels/image/random_resize_op.h" -#include "minddata/dataset/kernels/image/random_resize_with_bbox_op.h" -#include "minddata/dataset/kernels/image/random_rotation_op.h" -#include "minddata/dataset/kernels/image/random_select_subpolicy_op.h" -#include "minddata/dataset/kernels/image/random_vertical_flip_op.h" -#include "minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h" -#include "minddata/dataset/kernels/image/rescale_op.h" -#include "minddata/dataset/kernels/image/resize_bilinear_op.h" -#include "minddata/dataset/kernels/image/resize_op.h" -#include "minddata/dataset/kernels/image/resize_with_bbox_op.h" -#include "minddata/dataset/kernels/image/uniform_aug_op.h" -#include "minddata/dataset/kernels/no_op.h" -#include "minddata/dataset/kernels/py_func_op.h" -#include "minddata/dataset/kernels/random_apply_op.h" -#include "minddata/dataset/kernels/random_choice_op.h" -#include "minddata/dataset/text/kernels/jieba_tokenizer_op.h" -#include "minddata/dataset/text/kernels/lookup_op.h" -#include "minddata/dataset/text/kernels/ngram_op.h" -#include "minddata/dataset/text/kernels/sliding_window_op.h" -#include "minddata/dataset/text/kernels/to_number_op.h" -#include "minddata/dataset/text/kernels/unicode_char_tokenizer_op.h" -#include "minddata/dataset/text/kernels/wordpiece_tokenizer_op.h" -#include "minddata/dataset/text/kernels/sentence_piece_tokenizer_op.h" -#include "minddata/dataset/text/vocab.h" -#include "minddata/dataset/text/sentence_piece_vocab.h" -#include "minddata/dataset/util/random.h" -#include "minddata/mindrecord/include/shard_distributed_sample.h" -#include "minddata/mindrecord/include/shard_operator.h" -#include "minddata/mindrecord/include/shard_pk_sample.h" -#include "minddata/mindrecord/include/shard_sample.h" -#include "minddata/mindrecord/include/shard_sequential_sample.h" -#include "mindspore/ccsrc/minddata/dataset/text/kernels/truncate_sequence_pair_op.h" - -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" -#include "pybind11/stl_bind.h" - -#ifdef ENABLE_ICU4C -#include "minddata/dataset/text/kernels/basic_tokenizer_op.h" -#include "minddata/dataset/text/kernels/bert_tokenizer_op.h" -#include "minddata/dataset/text/kernels/case_fold_op.h" -#include "minddata/dataset/text/kernels/normalize_utf8_op.h" -#include "minddata/dataset/text/kernels/regex_replace_op.h" -#include "minddata/dataset/text/kernels/regex_tokenizer_op.h" -#include "minddata/dataset/text/kernels/unicode_script_tokenizer_op.h" -#include "minddata/dataset/text/kernels/whitespace_tokenizer_op.h" -#endif - -namespace py = pybind11; - -namespace mindspore { -namespace dataset { -#define THROW_IF_ERROR(s) \ - do { \ - Status rc = std::move(s); \ - if (rc.IsError()) throw std::runtime_error(rc.ToString()); \ - } while (false) - -Status PyListToTensorOps(const py::list &py_ops, std::vector> *ops) { - RETURN_UNEXPECTED_IF_NULL(ops); - for (auto op : py_ops) { - if (py::isinstance(op)) { - ops->emplace_back(op.cast>()); - } else if (py::isinstance(op)) { - ops->emplace_back(std::make_shared(op.cast())); - } else { - RETURN_STATUS_UNEXPECTED("element is neither a TensorOp nor a pyfunc."); - } - } - CHECK_FAIL_RETURN_UNEXPECTED(!ops->empty(), "TensorOp list is empty."); - for (auto const &op : *ops) { - RETURN_UNEXPECTED_IF_NULL(op); - } - return Status::OK(); -} - -void bindDEPipeline(py::module *m) { - (void)py::class_(*m, "DEPipeline") - .def(py::init<>()) - .def( - "AddNodeToTree", - [](DEPipeline &de, const OpName &op_name, const py::dict &args) { - py::dict out; - THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &out)); - return out; - }, - py::return_value_policy::reference) - .def_static("AddChildToParentNode", - [](const DsOpPtr &child_op, const DsOpPtr &parent_op) { - THROW_IF_ERROR(DEPipeline::AddChildToParentNode(child_op, parent_op)); - }) - .def("AssignRootNode", - [](DEPipeline &de, const DsOpPtr &dataset_op) { THROW_IF_ERROR(de.AssignRootNode(dataset_op)); }) - .def("SetBatchParameters", - [](DEPipeline &de, const py::dict &args) { THROW_IF_ERROR(de.SetBatchParameters(args)); }) - .def("LaunchTreeExec", [](DEPipeline &de, int32_t num_epochs) { THROW_IF_ERROR(de.LaunchTreeExec(num_epochs)); }) - .def("GetNextAsMap", - [](DEPipeline &de) { - py::dict out; - THROW_IF_ERROR(de.GetNextAsMap(&out)); - return out; - }) - .def("GetNextAsList", - [](DEPipeline &de) { - py::list out; - THROW_IF_ERROR(de.GetNextAsList(&out)); - return out; - }) - .def("GetOutputShapes", - [](DEPipeline &de) { - py::list out; - THROW_IF_ERROR(de.GetOutputShapes(&out)); - return out; - }) - .def("GetOutputTypes", - [](DEPipeline &de) { - py::list out; - THROW_IF_ERROR(de.GetOutputTypes(&out)); - return out; - }) - .def("GetDatasetSize", &DEPipeline::GetDatasetSize) - .def("GetBatchSize", &DEPipeline::GetBatchSize) - .def("GetNumClasses", &DEPipeline::GetNumClasses) - .def("GetRepeatCount", &DEPipeline::GetRepeatCount) - .def("StopSend", [](DEPipeline &de) { THROW_IF_ERROR(de.StopSend()); }) - .def("SaveDataset", [](DEPipeline &de, const std::vector &file_names, const std::string &file_type) { - THROW_IF_ERROR(de.SaveDataset(file_names, file_type)); - return true; - }); -} -void bindDatasetOps(py::module *m) { - (void)py::class_>(*m, "TFReaderOp") - .def_static("get_num_rows", [](const py::list &files, int64_t numParallelWorkers, bool estimate = false) { - int64_t count = 0; - std::vector filenames; - for (auto l : files) { - !l.is_none() ? filenames.push_back(py::str(l)) : (void)filenames.emplace_back(""); - } - THROW_IF_ERROR(TFReaderOp::CountTotalRows(&count, filenames, numParallelWorkers, estimate)); - return count; - }); - - (void)py::class_>(*m, "CifarOp") - .def_static("get_num_rows", [](const std::string &dir, bool isCifar10) { - int64_t count = 0; - THROW_IF_ERROR(CifarOp::CountTotalRows(dir, isCifar10, &count)); - return count; - }); - - (void)py::class_>(*m, "ImageFolderOp") - .def_static("get_num_rows_and_classes", [](const std::string &path) { - int64_t count = 0, num_classes = 0; - THROW_IF_ERROR(ImageFolderOp::CountRowsAndClasses(path, std::set{}, &count, &num_classes)); - return py::make_tuple(count, num_classes); - }); - - (void)py::class_>(*m, "MindRecordOp") - .def_static("get_num_rows", [](const std::vector &paths, bool load_dataset, const py::object &sampler, - const int64_t num_padded) { - int64_t count = 0; - std::shared_ptr op; - if (py::hasattr(sampler, "create_for_minddataset")) { - auto create = sampler.attr("create_for_minddataset"); - op = create().cast>(); - } - THROW_IF_ERROR(MindRecordOp::CountTotalRows(paths, load_dataset, op, &count, num_padded)); - return count; - }); - - (void)py::class_>(*m, "ManifestOp") - .def_static("get_num_rows_and_classes", - [](const std::string &file, const py::dict &dict, const std::string &usage) { - int64_t count = 0, num_classes = 0; - THROW_IF_ERROR(ManifestOp::CountTotalRows(file, dict, usage, &count, &num_classes)); - return py::make_tuple(count, num_classes); - }) - .def_static("get_class_indexing", [](const std::string &file, const py::dict &dict, const std::string &usage) { - std::map output_class_indexing; - THROW_IF_ERROR(ManifestOp::GetClassIndexing(file, dict, usage, &output_class_indexing)); - return output_class_indexing; - }); - - (void)py::class_>(*m, "MnistOp") - .def_static("get_num_rows", [](const std::string &dir) { - int64_t count = 0; - THROW_IF_ERROR(MnistOp::CountTotalRows(dir, &count)); - return count; - }); - - (void)py::class_>(*m, "TextFileOp") - .def_static("get_num_rows", [](const py::list &files) { - int64_t count = 0; - std::vector filenames; - for (auto file : files) { - !file.is_none() ? filenames.push_back(py::str(file)) : (void)filenames.emplace_back(""); - } - THROW_IF_ERROR(TextFileOp::CountAllFileRows(filenames, &count)); - return count; - }); - - (void)py::class_>(*m, "ClueOp") - .def_static("get_num_rows", [](const py::list &files) { - int64_t count = 0; - std::vector filenames; - for (auto file : files) { - file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file)); - } - THROW_IF_ERROR(ClueOp::CountAllFileRows(filenames, &count)); - return count; - }); - - (void)py::class_>(*m, "CsvOp") - .def_static("get_num_rows", [](const py::list &files, bool csv_header) { - int64_t count = 0; - std::vector filenames; - for (auto file : files) { - file.is_none() ? (void)filenames.emplace_back("") : filenames.push_back(py::str(file)); - } - THROW_IF_ERROR(CsvOp::CountAllFileRows(filenames, csv_header, &count)); - return count; - }); - - (void)py::class_>(*m, "VOCOp") - .def_static("get_num_rows", - [](const std::string &dir, const std::string &task_type, const std::string &task_mode, - const py::dict &dict, int64_t numSamples) { - int64_t count = 0; - THROW_IF_ERROR(VOCOp::CountTotalRows(dir, task_type, task_mode, dict, &count)); - return count; - }) - .def_static("get_class_indexing", [](const std::string &dir, const std::string &task_type, - const std::string &task_mode, const py::dict &dict) { - std::map output_class_indexing; - THROW_IF_ERROR(VOCOp::GetClassIndexing(dir, task_type, task_mode, dict, &output_class_indexing)); - return output_class_indexing; - }); - (void)py::class_>(*m, "CocoOp") - .def_static("get_class_indexing", - [](const std::string &dir, const std::string &file, const std::string &task) { - std::vector>> output_class_indexing; - THROW_IF_ERROR(CocoOp::GetClassIndexing(dir, file, task, &output_class_indexing)); - return output_class_indexing; - }) - .def_static("get_num_rows", [](const std::string &dir, const std::string &file, const std::string &task) { - int64_t count = 0; - THROW_IF_ERROR(CocoOp::CountTotalRows(dir, file, task, &count)); - return count; - }); -} -void bindTensor(py::module *m) { - (void)py::class_(*m, "GlobalContext") - .def_static("config_manager", &GlobalContext::config_manager, py::return_value_policy::reference); - - (void)py::class_>(*m, "ConfigManager") - .def("__str__", &ConfigManager::ToString) - .def("set_rows_per_buffer", &ConfigManager::set_rows_per_buffer) - .def("set_num_parallel_workers", &ConfigManager::set_num_parallel_workers) - .def("set_worker_connector_size", &ConfigManager::set_worker_connector_size) - .def("set_op_connector_size", &ConfigManager::set_op_connector_size) - .def("set_seed", &ConfigManager::set_seed) - .def("set_monitor_sampling_interval", &ConfigManager::set_monitor_sampling_interval) - .def("get_rows_per_buffer", &ConfigManager::rows_per_buffer) - .def("get_num_parallel_workers", &ConfigManager::num_parallel_workers) - .def("get_worker_connector_size", &ConfigManager::worker_connector_size) - .def("get_op_connector_size", &ConfigManager::op_connector_size) - .def("get_seed", &ConfigManager::seed) - .def("get_monitor_sampling_interval", &ConfigManager::monitor_sampling_interval) - .def("load", [](ConfigManager &c, std::string s) { THROW_IF_ERROR(c.LoadFile(s)); }); - - (void)py::class_>(*m, "Tensor", py::buffer_protocol()) - .def(py::init([](py::array arr) { - std::shared_ptr out; - THROW_IF_ERROR(Tensor::CreateFromNpArray(arr, &out)); - return out; - })) - .def_buffer([](Tensor &tensor) { - py::buffer_info info; - THROW_IF_ERROR(Tensor::GetBufferInfo(&tensor, &info)); - return info; - }) - .def("__str__", &Tensor::ToString) - .def("shape", &Tensor::shape) - .def("type", &Tensor::type) - .def("as_array", [](py::object &t) { - auto &tensor = py::cast(t); - if (tensor.type() == DataType::DE_STRING) { - py::array res; - tensor.GetDataAsNumpyStrings(&res); - return res; - } - py::buffer_info info; - THROW_IF_ERROR(Tensor::GetBufferInfo(&tensor, &info)); - return py::array(pybind11::dtype(info), info.shape, info.strides, info.ptr, t); - }); - - (void)py::class_(*m, "TensorShape") - .def(py::init([](const py::list &list) { - std::vector list_c; - for (auto &i : list) { - if (!i.is_none()) { - list_c.push_back(i.cast()); - } else { - list_c.push_back(TensorShape::kDimUnknown); - } - } - TensorShape out(list_c); - return out; - })) - .def("__str__", &TensorShape::ToString) - .def("as_list", &TensorShape::AsPyList) - .def("is_known", &TensorShape::known); - - (void)py::class_(*m, "DataType") - .def(py::init()) - .def(py::self == py::self) - .def("__str__", &DataType::ToString) - .def("__deepcopy__", [](py::object &t, py::dict memo) { return t; }); -} - -void bindTensorOps1(py::module *m) { - (void)py::class_>(*m, "TensorOp") - .def("__deepcopy__", [](py::object &t, py::dict memo) { return t; }); - - (void)py::class_>( - *m, "AutoContrastOp", "Tensor operation to apply autocontrast on an image.") - .def(py::init>(), py::arg("cutoff") = AutoContrastOp::kCutOff, - py::arg("ignore") = AutoContrastOp::kIgnore); - - (void)py::class_>( - *m, "NormalizeOp", "Tensor operation to normalize an image. Takes mean and std.") - .def(py::init(), py::arg("meanR"), py::arg("meanG"), py::arg("meanB"), - py::arg("stdR"), py::arg("stdG"), py::arg("stdB")); - - (void)py::class_>( - *m, "EqualizeOp", "Tensor operation to apply histogram equalization on images.") - .def(py::init<>()); - - (void)py::class_>(*m, "InvertOp", - "Tensor operation to apply invert on RGB images.") - .def(py::init<>()); - - (void)py::class_>( - *m, "RescaleOp", "Tensor operation to rescale an image. Takes scale and shift.") - .def(py::init(), py::arg("rescale"), py::arg("shift")); - - (void)py::class_>( - *m, "CenterCropOp", "Tensor operation to crop and image in the middle. Takes height and width (optional)") - .def(py::init(), py::arg("height"), py::arg("width") = CenterCropOp::kDefWidth); - - (void)py::class_>( - *m, "ResizeOp", "Tensor operation to resize an image. Takes height, width and mode") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth") = ResizeOp::kDefWidth, py::arg("interpolation") = ResizeOp::kDefInterpolation); - - (void)py::class_>( - *m, "ResizeWithBBoxOp", "Tensor operation to resize an image. Takes height, width and mode.") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth") = ResizeWithBBoxOp::kDefWidth, - py::arg("interpolation") = ResizeWithBBoxOp::kDefInterpolation); - - (void)py::class_>( - *m, "RandomResizeWithBBoxOp", - "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth") = RandomResizeWithBBoxOp::kDefTargetWidth); - - (void)py::class_>( - *m, "UniformAugOp", "Tensor operation to apply random augmentation(s).") - .def(py::init>, int32_t>(), py::arg("transforms"), - py::arg("NumOps") = UniformAugOp::kDefNumOps); - - (void)py::class_>( - *m, "BoundingBoxAugmentOp", "Tensor operation to apply a transformation on a random choice of bounding boxes.") - .def(py::init, float>(), py::arg("transform"), - py::arg("ratio") = BoundingBoxAugmentOp::kDefRatio); - - (void)py::class_>( - *m, "ResizeBilinearOp", - "Tensor operation to resize an image using " - "Bilinear mode. Takes height and width.") - .def(py::init(), py::arg("targetHeight"), py::arg("targetWidth") = ResizeBilinearOp::kDefWidth); - - (void)py::class_>(*m, "DecodeOp", - "Tensor operation to decode a jpg image") - .def(py::init<>()) - .def(py::init(), py::arg("rgb_format") = DecodeOp::kDefRgbFormat); - - (void)py::class_>( - *m, "RandomHorizontalFlipOp", "Tensor operation to randomly flip an image horizontally.") - .def(py::init(), py::arg("probability") = RandomHorizontalFlipOp::kDefProbability); - - (void)py::class_>( - *m, "RandomHorizontalFlipWithBBoxOp", - "Tensor operation to randomly flip an image horizontally, while flipping bounding boxes.") - .def(py::init(), py::arg("probability") = RandomHorizontalFlipWithBBoxOp::kDefProbability); -} - -void bindTensorOps2(py::module *m) { - (void)py::class_>( - *m, "RandomVerticalFlipOp", "Tensor operation to randomly flip an image vertically.") - .def(py::init(), py::arg("probability") = RandomVerticalFlipOp::kDefProbability); - - (void)py::class_>( - *m, "RandomVerticalFlipWithBBoxOp", - "Tensor operation to randomly flip an image vertically" - " and adjust bounding boxes.") - .def(py::init(), py::arg("probability") = RandomVerticalFlipWithBBoxOp::kDefProbability); - - (void)py::class_>(*m, "RandomCropOp", - "Gives random crop of specified size " - "Takes crop size") - .def(py::init(), - py::arg("cropHeight"), py::arg("cropWidth"), py::arg("padTop") = RandomCropOp::kDefPadTop, - py::arg("padBottom") = RandomCropOp::kDefPadBottom, py::arg("padLeft") = RandomCropOp::kDefPadLeft, - py::arg("padRight") = RandomCropOp::kDefPadRight, py::arg("borderType") = RandomCropOp::kDefBorderType, - py::arg("padIfNeeded") = RandomCropOp::kDefPadIfNeeded, py::arg("fillR") = RandomCropOp::kDefFillR, - py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); - (void)py::class_>(*m, "ChannelSwapOp").def(py::init<>()); - - (void)py::class_>(*m, "RandomCropWithBBoxOp", - "Gives random crop of given " - "size + adjusts bboxes " - "Takes crop size") - .def(py::init(), - py::arg("cropHeight"), py::arg("cropWidth"), py::arg("padTop") = RandomCropWithBBoxOp::kDefPadTop, - py::arg("padBottom") = RandomCropWithBBoxOp::kDefPadBottom, - py::arg("padLeft") = RandomCropWithBBoxOp::kDefPadLeft, - py::arg("padRight") = RandomCropWithBBoxOp::kDefPadRight, - py::arg("borderType") = RandomCropWithBBoxOp::kDefBorderType, - py::arg("padIfNeeded") = RandomCropWithBBoxOp::kDefPadIfNeeded, - py::arg("fillR") = RandomCropWithBBoxOp::kDefFillR, py::arg("fillG") = RandomCropWithBBoxOp::kDefFillG, - py::arg("fillB") = RandomCropWithBBoxOp::kDefFillB); - - (void)py::class_>( - *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") - .def(py::init()); - - (void)py::class_>( - *m, "FillOp", "Tensor operation to return tensor filled with same value as input fill value.") - .def(py::init>()); - - (void)py::class_>(*m, "SliceOp", "Tensor slice operation.") - .def(py::init()) - .def(py::init([](const py::list &py_list) { - std::vector c_list; - for (auto l : py_list) { - if (!l.is_none()) { - c_list.push_back(py::reinterpret_borrow(l)); - } - } - return std::make_shared(c_list); - })) - .def(py::init([](const py::tuple &py_slice) { - if (py_slice.size() != 3) { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Wrong slice object")); - } - Slice c_slice; - if (!py_slice[0].is_none() && !py_slice[1].is_none() && !py_slice[2].is_none()) { - c_slice = Slice(py::reinterpret_borrow(py_slice[0]), py::reinterpret_borrow(py_slice[1]), - py::reinterpret_borrow(py_slice[2])); - } else if (py_slice[0].is_none() && py_slice[2].is_none()) { - c_slice = Slice(py::reinterpret_borrow(py_slice[1])); - } else if (!py_slice[0].is_none() && !py_slice[1].is_none()) { - c_slice = Slice(py::reinterpret_borrow(py_slice[0]), py::reinterpret_borrow(py_slice[1])); - } - - if (!c_slice.valid()) { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Wrong slice object")); - } - return std::make_shared(c_slice); - })); - - (void)py::enum_(*m, "RelationalOp", py::arithmetic()) - .value("EQ", RelationalOp::kEqual) - .value("NE", RelationalOp::kNotEqual) - .value("LT", RelationalOp::kLess) - .value("LE", RelationalOp::kLessEqual) - .value("GT", RelationalOp::kGreater) - .value("GE", RelationalOp::kGreaterEqual) - .export_values(); - - (void)py::class_>(*m, "MaskOp", - "Tensor mask operation using relational comparator") - .def(py::init, DataType>()); - - (void)py::class_>(*m, "DuplicateOp", "Duplicate tensor.") - .def(py::init<>()); - - (void)py::class_>( - *m, "TruncateSequencePairOp", "Tensor operation to truncate two tensors to a max_length") - .def(py::init()); - - (void)py::class_>(*m, "ConcatenateOp", - "Tensor operation concatenate tensors.") - .def(py::init, std::shared_ptr>(), py::arg("axis"), - py::arg("prepend").none(true), py::arg("append").none(true)); - - (void)py::class_>( - *m, "RandomRotationOp", - "Tensor operation to apply RandomRotation." - "Takes a range for degrees and " - "optional parameters for rotation center and image expand") - .def(py::init(), - py::arg("startDegree"), py::arg("endDegree"), py::arg("centerX") = RandomRotationOp::kDefCenterX, - py::arg("centerY") = RandomRotationOp::kDefCenterY, - py::arg("interpolation") = RandomRotationOp::kDefInterpolation, - py::arg("expand") = RandomRotationOp::kDefExpand, py::arg("fillR") = RandomRotationOp::kDefFillR, - py::arg("fillG") = RandomRotationOp::kDefFillG, py::arg("fillB") = RandomRotationOp::kDefFillB); - - (void)py::class_>( - *m, "PadEndOp", "Tensor operation to pad end of tensor with a pad value.") - .def(py::init>()); -} - -void bindTensorOps3(py::module *m) { - (void)py::class_>( - *m, "RandomCropAndResizeOp", - "Tensor operation to randomly crop an image and resize to a given size." - "Takes output height and width and" - "optional parameters for lower and upper bound for aspect ratio (h/w) and scale," - "interpolation mode, and max attempts to crop") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth"), py::arg("scaleLb") = RandomCropAndResizeOp::kDefScaleLb, - py::arg("scaleUb") = RandomCropAndResizeOp::kDefScaleUb, - py::arg("aspectLb") = RandomCropAndResizeOp::kDefAspectLb, - py::arg("aspectUb") = RandomCropAndResizeOp::kDefAspectUb, - py::arg("interpolation") = RandomCropAndResizeOp::kDefInterpolation, - py::arg("maxIter") = RandomCropAndResizeOp::kDefMaxIter); - - (void)py::class_>( - *m, "RandomCropAndResizeWithBBoxOp", - "Tensor operation to randomly crop an image (with BBoxes) and resize to a given size." - "Takes output height and width and" - "optional parameters for lower and upper bound for aspect ratio (h/w) and scale," - "interpolation mode, and max attempts to crop") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth"), py::arg("scaleLb") = RandomCropAndResizeWithBBoxOp::kDefScaleLb, - py::arg("scaleUb") = RandomCropAndResizeWithBBoxOp::kDefScaleUb, - py::arg("aspectLb") = RandomCropAndResizeWithBBoxOp::kDefAspectLb, - py::arg("aspectUb") = RandomCropAndResizeWithBBoxOp::kDefAspectUb, - py::arg("interpolation") = RandomCropAndResizeWithBBoxOp::kDefInterpolation, - py::arg("maxIter") = RandomCropAndResizeWithBBoxOp::kDefMaxIter); - - (void)py::class_>( - *m, "RandomColorAdjustOp", - "Tensor operation to adjust an image's color randomly." - "Takes range for brightness, contrast, saturation, hue and") - .def(py::init(), py::arg("bright_factor_start"), - py::arg("bright_factor_end"), py::arg("contrast_factor_start"), py::arg("contrast_factor_end"), - py::arg("saturation_factor_start"), py::arg("saturation_factor_end"), py::arg("hue_factor_start"), - py::arg("hue_factor_end")); - - (void)py::class_>( - *m, "RandomResizeOp", - "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth") = RandomResizeOp::kDefTargetWidth); - - (void)py::class_>( - *m, "CutOutOp", "Tensor operation to randomly erase a portion of the image. Takes height and width.") - .def(py::init(), py::arg("boxHeight"), - py::arg("boxWidth"), py::arg("numPatches"), py::arg("randomColor") = CutOutOp::kDefRandomColor, - py::arg("fillR") = CutOutOp::kDefFillR, py::arg("fillG") = CutOutOp::kDefFillG, - py::arg("fillB") = CutOutOp::kDefFillB); -} - -void bindTensorOps4(py::module *m) { - (void)py::class_>( - *m, "TypeCastOp", "Tensor operator to type cast data to a specified type.") - .def(py::init(), py::arg("data_type")) - .def(py::init(), py::arg("data_type")); - - (void)py::class_>(*m, "NoOp", - "TensorOp that does nothing, for testing purposes only.") - .def(py::init<>()); - - (void)py::class_>( - *m, "ToFloat16Op", py::dynamic_attr(), "Tensor operator to type cast float32 data to a float16 type.") - .def(py::init<>()); - - (void)py::class_>( - *m, "RandomCropDecodeResizeOp", "equivalent to RandomCropAndResize but crops before decoding") - .def(py::init(), py::arg("targetHeight"), - py::arg("targetWidth"), py::arg("scaleLb") = RandomCropDecodeResizeOp::kDefScaleLb, - py::arg("scaleUb") = RandomCropDecodeResizeOp::kDefScaleUb, - py::arg("aspectLb") = RandomCropDecodeResizeOp::kDefAspectLb, - py::arg("aspectUb") = RandomCropDecodeResizeOp::kDefAspectUb, - py::arg("interpolation") = RandomCropDecodeResizeOp::kDefInterpolation, - py::arg("maxIter") = RandomCropDecodeResizeOp::kDefMaxIter); - - (void)py::class_>( - *m, "PadOp", - "Pads image with specified color, default black, " - "Takes amount to pad for top, bottom, left, right of image, boarder type and color") - .def(py::init(), py::arg("padTop"), - py::arg("padBottom"), py::arg("padLeft"), py::arg("padRight"), py::arg("borderTypes") = PadOp::kDefBorderType, - py::arg("fillR") = PadOp::kDefFillR, py::arg("fillG") = PadOp::kDefFillG, py::arg("fillB") = PadOp::kDefFillB); - (void)py::class_>(*m, "ToNumberOp", - "TensorOp to convert strings to numbers.") - .def(py::init(), py::arg("data_type")) - .def(py::init(), py::arg("data_type")); -} - -void bindTokenizerOps(py::module *m) { - (void)py::class_>(*m, "JiebaTokenizerOp", "") - .def(py::init(), py::arg("hmm_path"), - py::arg("mp_path"), py::arg("mode") = JiebaMode::kMix, - py::arg("with_offsets") = JiebaTokenizerOp::kDefWithOffsets) - .def("add_word", - [](JiebaTokenizerOp &self, const std::string word, int freq) { THROW_IF_ERROR(self.AddWord(word, freq)); }); - (void)py::class_>( - *m, "UnicodeCharTokenizerOp", "Tokenize a scalar tensor of UTF-8 string to Unicode characters.") - .def(py::init(), py::arg("with_offsets") = UnicodeCharTokenizerOp::kDefWithOffsets); - (void)py::class_>(*m, "LookupOp", - "Tensor operation to LookUp each word.") - .def(py::init([](std::shared_ptr vocab, const py::object &py_word) { - if (vocab == nullptr) { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "vocab object type is incorrect or null.")); - } - if (py_word.is_none()) { - return std::make_shared(vocab, Vocab::kNoTokenExists); - } - std::string word = py::reinterpret_borrow(py_word); - WordIdType default_id = vocab->Lookup(word); - if (default_id == Vocab::kNoTokenExists) { - THROW_IF_ERROR( - Status(StatusCode::kUnexpectedError, "default unknown token: " + word + " doesn't exist in vocab.")); - } - return std::make_shared(vocab, default_id); - })); - (void)py::class_>(*m, "NgramOp", "TensorOp performs ngram mapping.") - .def(py::init &, int32_t, int32_t, const std::string &, const std::string &, - const std::string &>(), - py::arg("ngrams"), py::arg("l_pad_len"), py::arg("r_pad_len"), py::arg("l_pad_token"), py::arg("r_pad_token"), - py::arg("separator")); - (void)py::class_>( - *m, "WordpieceTokenizerOp", "Tokenize scalar token or 1-D tokens to subword tokens.") - .def( - py::init &, const std::string &, const int &, const std::string &, const bool &>(), - py::arg("vocab"), py::arg("suffix_indicator") = std::string(WordpieceTokenizerOp::kDefSuffixIndicator), - py::arg("max_bytes_per_token") = WordpieceTokenizerOp::kDefMaxBytesPerToken, - py::arg("unknown_token") = std::string(WordpieceTokenizerOp::kDefUnknownToken), - py::arg("with_offsets") = WordpieceTokenizerOp::kDefWithOffsets); - (void)py::class_>( - *m, "SlidingWindowOp", "TensorOp to apply sliding window to a 1-D Tensor.") - .def(py::init(), py::arg("width"), py::arg("axis")); - (void)py::class_>( - *m, "SentencePieceTokenizerOp", "Tokenize scalar token or 1-D tokens to tokens by sentence piece.") - .def(py::init &, const SPieceTokenizerLoadType, const SPieceTokenizerOutType>(), - py::arg("vocab"), py::arg("load_type") = SPieceTokenizerLoadType::kModel, - py::arg("out_type") = SPieceTokenizerOutType::kString) - .def( - py::init(), - py::arg("model_path"), py::arg("model_filename"), py::arg("load_type") = SPieceTokenizerLoadType::kFile, - py::arg("out_type") = SPieceTokenizerOutType::kString); -} - -void bindDependIcuTokenizerOps(py::module *m) { -#ifdef ENABLE_ICU4C - (void)py::class_>( - *m, "WhitespaceTokenizerOp", "Tokenize a scalar tensor of UTF-8 string on ICU defined whitespaces.") - .def(py::init(), py::arg("with_offsets") = WhitespaceTokenizerOp::kDefWithOffsets); - (void)py::class_>( - *m, "UnicodeScriptTokenizerOp", "Tokenize a scalar tensor of UTF-8 string on Unicode script boundaries.") - .def(py::init<>()) - .def(py::init(), - py::arg("keep_whitespace") = UnicodeScriptTokenizerOp::kDefKeepWhitespace, - py::arg("with_offsets") = UnicodeScriptTokenizerOp::kDefWithOffsets); - (void)py::class_>( - *m, "CaseFoldOp", "Apply case fold operation on utf-8 string tensor") - .def(py::init<>()); - (void)py::class_>( - *m, "NormalizeUTF8Op", "Apply normalize operation on utf-8 string tensor.") - .def(py::init<>()) - .def(py::init(), py::arg("normalize_form") = NormalizeUTF8Op::kDefNormalizeForm); - (void)py::class_>( - *m, "RegexReplaceOp", "Replace utf-8 string tensor with 'replace' according to regular expression 'pattern'.") - .def(py::init(), py::arg("pattern"), py::arg("replace"), - py::arg("replace_all")); - (void)py::class_>( - *m, "RegexTokenizerOp", "Tokenize a scalar tensor of UTF-8 string by regex expression pattern.") - .def(py::init(), py::arg("delim_pattern"), - py::arg("keep_delim_pattern"), py::arg("with_offsets") = RegexTokenizerOp::kDefWithOffsets); - (void)py::class_>( - *m, "BasicTokenizerOp", "Tokenize a scalar tensor of UTF-8 string by specific rules.") - .def(py::init(), - py::arg("lower_case") = BasicTokenizerOp::kDefLowerCase, - py::arg("keep_whitespace") = BasicTokenizerOp::kDefKeepWhitespace, - py::arg("normalization_form") = BasicTokenizerOp::kDefNormalizationForm, - py::arg("preserve_unused_token") = BasicTokenizerOp::kDefPreserveUnusedToken, - py::arg("with_offsets") = BasicTokenizerOp::kDefWithOffsets); - (void)py::class_>(*m, "BertTokenizerOp", - "Tokenizer used for Bert text process.") - .def(py::init &, const std::string &, const int &, const std::string &, const bool &, - const bool &, const NormalizeForm &, const bool &, const bool &>(), - py::arg("vocab"), py::arg("suffix_indicator") = std::string(WordpieceTokenizerOp::kDefSuffixIndicator), - py::arg("max_bytes_per_token") = WordpieceTokenizerOp::kDefMaxBytesPerToken, - py::arg("unknown_token") = std::string(WordpieceTokenizerOp::kDefUnknownToken), - py::arg("lower_case") = BasicTokenizerOp::kDefLowerCase, - py::arg("keep_whitespace") = BasicTokenizerOp::kDefKeepWhitespace, - py::arg("normalization_form") = BasicTokenizerOp::kDefNormalizationForm, - py::arg("preserve_unused_token") = BasicTokenizerOp::kDefPreserveUnusedToken, - py::arg("with_offsets") = WordpieceTokenizerOp::kDefWithOffsets); -#endif -} - -void bindSamplerOps(py::module *m) { - (void)py::class_>(*m, "Sampler") - .def("set_num_rows", [](Sampler &self, int64_t rows) { THROW_IF_ERROR(self.SetNumRowsInDataset(rows)); }) - .def("set_num_samples", [](Sampler &self, int64_t samples) { THROW_IF_ERROR(self.SetNumSamples(samples)); }) - .def("initialize", [](Sampler &self) { THROW_IF_ERROR(self.InitSampler()); }) - .def("get_indices", - [](Sampler &self) { - py::array ret; - THROW_IF_ERROR(self.GetAllIdsThenReset(&ret)); - return ret; - }) - .def("add_child", - [](std::shared_ptr self, std::shared_ptr child) { THROW_IF_ERROR(self->AddChild(child)); }); - - (void)py::class_>(*m, "ShardOperator") - .def("add_child", [](std::shared_ptr self, - std::shared_ptr child) { self->SetChildOp(child); }); - - (void)py::class_>(*m, "DistributedSampler") - .def(py::init()); - - (void)py::class_>(*m, "PKSampler") - .def(py::init()); - - (void)py::class_>(*m, "RandomSampler") - .def(py::init()); - - (void)py::class_>(*m, "SequentialSampler") - .def(py::init()); - - (void)py::class_>(*m, "SubsetRandomSampler") - .def(py::init>()); - - (void)py::class_>( - *m, "MindrecordSubsetRandomSampler") - .def(py::init, uint32_t>(), py::arg("indices"), py::arg("seed") = GetSeed()); - - (void)py::class_>( - *m, "MindrecordPkSampler") - .def(py::init([](int64_t kVal, std::string kColumn, bool shuffle) { - if (shuffle == true) { - return std::make_shared(kColumn, kVal, std::numeric_limits::max(), - GetSeed()); - } else { - return std::make_shared(kColumn, kVal); - } - })); - - (void)py::class_>(*m, "MindrecordDistributedSampler") - .def(py::init()); - - (void)py::class_>( - *m, "MindrecordRandomSampler") - .def(py::init([](int64_t num_samples, bool replacement, bool reshuffle_each_epoch) { - return std::make_shared(GetSeed(), num_samples, replacement, reshuffle_each_epoch); - })); - - (void)py::class_>(*m, "MindrecordSequentialSampler") - .def(py::init([](int num_samples, int start_index) { - return std::make_shared(num_samples, start_index); - })); - - (void)py::class_>(*m, "WeightedRandomSampler") - .def(py::init, bool>()); - - (void)py::class_>(*m, "PythonSampler") - .def(py::init()); -} - -void bindInfoObjects(py::module *m) { - (void)py::class_(*m, "CBatchInfo") - .def(py::init()) - .def("get_epoch_num", &BatchOp::CBatchInfo::get_epoch_num) - .def("get_batch_num", &BatchOp::CBatchInfo::get_batch_num); -} - -void bindCacheClient(py::module *m) { - (void)py::class_>(*m, "CacheClient") - .def(py::init()); -} - -void bindVocabObjects(py::module *m) { - (void)py::class_>(*m, "Vocab") - .def(py::init<>()) - .def_static("from_list", - [](const py::list &words, const py::list &special_tokens, bool special_first) { - std::shared_ptr v; - THROW_IF_ERROR(Vocab::BuildFromPyList(words, special_tokens, special_first, &v)); - return v; - }) - .def_static("from_file", - [](const std::string &path, const std::string &dlm, int32_t vocab_size, const py::list &special_tokens, - bool special_first) { - std::shared_ptr v; - THROW_IF_ERROR(Vocab::BuildFromFile(path, dlm, vocab_size, special_tokens, special_first, &v)); - return v; - }) - .def_static("from_dict", [](const py::dict &words) { - std::shared_ptr v; - THROW_IF_ERROR(Vocab::BuildFromPyDict(words, &v)); - return v; - }); - (void)py::class_>(*m, "SentencePieceVocab") - .def(py::init<>()) - .def_static("from_file", - [](const py::list &paths, const int vocab_size, const float character_coverage, - const SentencePieceModel model_type, const py::dict ¶ms) { - std::shared_ptr v; - std::vector path_list; - for (auto path : paths) { - path_list.emplace_back(py::str(path)); - } - std::unordered_map param_map; - for (auto param : params) { - std::string key = py::reinterpret_borrow(param.first); - if (key == "input" || key == "vocab_size" || key == "model_prefix" || key == "character_coverage" || - key == "model_type") { - continue; - } - param_map[key] = py::reinterpret_borrow(param.second); - } - THROW_IF_ERROR(SentencePieceVocab::BuildFromFile(path_list, vocab_size, character_coverage, - model_type, param_map, &v)); - return v; - }) - .def_static("save_model", - [](const std::shared_ptr *vocab, std::string path, std::string filename) { - THROW_IF_ERROR(SentencePieceVocab::SaveModel(vocab, path, filename)); - }); -} - -void bindGraphData(py::module *m) { - (void)py::class_>(*m, "Graph") - .def(py::init([](std::string dataset_file, int32_t num_workers) { - std::shared_ptr g_out = std::make_shared(dataset_file, num_workers); - THROW_IF_ERROR(g_out->Init()); - return g_out; - })) - .def("get_all_nodes", - [](gnn::Graph &g, gnn::NodeType node_type) { - std::shared_ptr out; - THROW_IF_ERROR(g.GetAllNodes(node_type, &out)); - return out; - }) - .def("get_all_edges", - [](gnn::Graph &g, gnn::EdgeType edge_type) { - std::shared_ptr out; - THROW_IF_ERROR(g.GetAllEdges(edge_type, &out)); - return out; - }) - .def("get_nodes_from_edges", - [](gnn::Graph &g, std::vector edge_list) { - std::shared_ptr out; - THROW_IF_ERROR(g.GetNodesFromEdges(edge_list, &out)); - return out; - }) - .def("get_all_neighbors", - [](gnn::Graph &g, std::vector node_list, gnn::NodeType neighbor_type) { - std::shared_ptr out; - THROW_IF_ERROR(g.GetAllNeighbors(node_list, neighbor_type, &out)); - return out; - }) - .def("get_sampled_neighbors", - [](gnn::Graph &g, std::vector node_list, std::vector neighbor_nums, - std::vector neighbor_types) { - std::shared_ptr out; - THROW_IF_ERROR(g.GetSampledNeighbors(node_list, neighbor_nums, neighbor_types, &out)); - return out; - }) - .def("get_neg_sampled_neighbors", - [](gnn::Graph &g, std::vector node_list, gnn::NodeIdType neighbor_num, - gnn::NodeType neg_neighbor_type) { - std::shared_ptr out; - THROW_IF_ERROR(g.GetNegSampledNeighbors(node_list, neighbor_num, neg_neighbor_type, &out)); - return out; - }) - .def("get_node_feature", - [](gnn::Graph &g, std::shared_ptr node_list, std::vector feature_types) { - TensorRow out; - THROW_IF_ERROR(g.GetNodeFeature(node_list, feature_types, &out)); - return out.getRow(); - }) - .def("get_edge_feature", - [](gnn::Graph &g, std::shared_ptr edge_list, std::vector feature_types) { - TensorRow out; - THROW_IF_ERROR(g.GetEdgeFeature(edge_list, feature_types, &out)); - return out.getRow(); - }) - .def("graph_info", - [](gnn::Graph &g) { - py::dict out; - THROW_IF_ERROR(g.GraphInfo(&out)); - return out; - }) - .def("random_walk", [](gnn::Graph &g, std::vector node_list, std::vector meta_path, - float step_home_param, float step_away_param, gnn::NodeIdType default_node) { - std::shared_ptr out; - THROW_IF_ERROR(g.RandomWalk(node_list, meta_path, step_home_param, step_away_param, default_node, &out)); - return out; - }); -} - -void bindRandomTransformTensorOps(py::module *m) { - (void)py::class_>(*m, "ComposeOp") - .def(py::init([](const py::list &ops) { - std::vector> t_ops; - THROW_IF_ERROR(PyListToTensorOps(ops, &t_ops)); - return std::make_shared(t_ops); - })); - (void)py::class_>(*m, "RandomChoiceOp") - .def(py::init([](const py::list &ops) { - std::vector> t_ops; - THROW_IF_ERROR(PyListToTensorOps(ops, &t_ops)); - return std::make_shared(t_ops); - })); - (void)py::class_>(*m, "RandomApplyOp") - .def(py::init([](double prob, const py::list &ops) { - std::vector> t_ops; - THROW_IF_ERROR(PyListToTensorOps(ops, &t_ops)); - if (prob < 0 || prob > 1) { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "prob needs to be within [0,1].")); - } - return std::make_shared(prob, t_ops); - })); - (void)py::class_>( - *m, "RandomSelectSubpolicyOp") - .def(py::init([](const py::list &py_policy) { - std::vector cpp_policy; - for (auto &py_sub : py_policy) { - cpp_policy.push_back({}); - for (auto handle : py_sub.cast()) { - py::tuple tp = handle.cast(); - if (tp.is_none() || tp.size() != 2) { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "Each tuple in subpolicy should be (op, prob).")); - } - std::shared_ptr t_op; - if (py::isinstance(tp[0])) { - t_op = (tp[0]).cast>(); - } else if (py::isinstance(tp[0])) { - t_op = std::make_shared((tp[0]).cast()); - } else { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "op is neither a tensorOp nor a pyfunc.")); - } - double prob = (tp[1]).cast(); - if (prob < 0 || prob > 1) { - THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "prob needs to be with [0,1].")); - } - cpp_policy.back().emplace_back(std::make_pair(t_op, prob)); - } - } - return std::make_shared(cpp_policy); - })); -} - -// This is where we externalize the C logic as python modules -PYBIND11_MODULE(_c_dataengine, m) { - m.doc() = "pybind11 for _c_dataengine"; - (void)py::class_>(m, "DatasetOp"); - - (void)py::enum_(m, "OpName", py::arithmetic()) - .value("SHUFFLE", OpName::kShuffle) - .value("BATCH", OpName::kBatch) - .value("BUCKETBATCH", OpName::kBucketBatch) - .value("BARRIER", OpName::kBarrier) - .value("MINDRECORD", OpName::kMindrecord) - .value("CACHE", OpName::kCache) - .value("REPEAT", OpName::kRepeat) - .value("SKIP", OpName::kSkip) - .value("TAKE", OpName::kTake) - .value("ZIP", OpName::kZip) - .value("CONCAT", OpName::kConcat) - .value("MAP", OpName::kMap) - .value("FILTER", OpName::kFilter) - .value("DEVICEQUEUE", OpName::kDeviceQueue) - .value("GENERATOR", OpName::kGenerator) - .export_values() - .value("RENAME", OpName::kRename) - .value("TFREADER", OpName::kTfReader) - .value("PROJECT", OpName::kProject) - .value("IMAGEFOLDER", OpName::kImageFolder) - .value("MNIST", OpName::kMnist) - .value("MANIFEST", OpName::kManifest) - .value("VOC", OpName::kVoc) - .value("COCO", OpName::kCoco) - .value("CIFAR10", OpName::kCifar10) - .value("CIFAR100", OpName::kCifar100) - .value("RANDOMDATA", OpName::kRandomData) - .value("BUILDVOCAB", OpName::kBuildVocab) - .value("SENTENCEPIECEVOCAB", OpName::kSentencePieceVocab) - .value("CELEBA", OpName::kCelebA) - .value("TEXTFILE", OpName::kTextFile) - .value("EPOCHCTRL", OpName::kEpochCtrl) - .value("CSV", OpName::kCsv) - .value("CLUE", OpName::kClue); - - (void)py::enum_(m, "JiebaMode", py::arithmetic()) - .value("DE_JIEBA_MIX", JiebaMode::kMix) - .value("DE_JIEBA_MP", JiebaMode::kMp) - .value("DE_JIEBA_HMM", JiebaMode::kHmm) - .export_values(); - -#ifdef ENABLE_ICU4C - (void)py::enum_(m, "NormalizeForm", py::arithmetic()) - .value("DE_NORMALIZE_NONE", NormalizeForm::kNone) - .value("DE_NORMALIZE_NFC", NormalizeForm::kNfc) - .value("DE_NORMALIZE_NFKC", NormalizeForm::kNfkc) - .value("DE_NORMALIZE_NFD", NormalizeForm::kNfd) - .value("DE_NORMALIZE_NFKD", NormalizeForm::kNfkd) - .export_values(); -#endif - - (void)py::enum_(m, "InterpolationMode", py::arithmetic()) - .value("DE_INTER_LINEAR", InterpolationMode::kLinear) - .value("DE_INTER_CUBIC", InterpolationMode::kCubic) - .value("DE_INTER_AREA", InterpolationMode::kArea) - .value("DE_INTER_NEAREST_NEIGHBOUR", InterpolationMode::kNearestNeighbour) - .export_values(); - - (void)py::enum_(m, "BorderType", py::arithmetic()) - .value("DE_BORDER_CONSTANT", BorderType::kConstant) - .value("DE_BORDER_EDGE", BorderType::kEdge) - .value("DE_BORDER_REFLECT", BorderType::kReflect) - .value("DE_BORDER_SYMMETRIC", BorderType::kSymmetric) - .export_values(); - - (void)py::enum_(m, "SentencePieceModel", py::arithmetic()) - .value("DE_SENTENCE_PIECE_UNIGRAM", SentencePieceModel::kUnigram) - .value("DE_SENTENCE_PIECE_BPE", SentencePieceModel::kBpe) - .value("DE_SENTENCE_PIECE_CHAR", SentencePieceModel::kChar) - .value("DE_SENTENCE_PIECE_WORD", SentencePieceModel::kWord) - .export_values(); - - (void)py::enum_(m, "SPieceTokenizerOutType", py::arithmetic()) - .value("DE_SPIECE_TOKENIZER_OUTTYPE_KString", SPieceTokenizerOutType::kString) - .value("DE_SPIECE_TOKENIZER_OUTTYPE_KINT", SPieceTokenizerOutType::kInt) - .export_values(); - - (void)py::enum_(m, "SPieceTokenizerLoadType", py::arithmetic()) - .value("DE_SPIECE_TOKENIZER_LOAD_KFILE", SPieceTokenizerLoadType::kFile) - .value("DE_SPIECE_TOKENIZER_LOAD_KMODEL", SPieceTokenizerLoadType::kModel) - .export_values(); - - bindDEPipeline(&m); - bindTensor(&m); - bindTensorOps1(&m); - bindTensorOps2(&m); - bindTensorOps3(&m); - bindTensorOps4(&m); - bindTokenizerOps(&m); - bindSamplerOps(&m); - bindDatasetOps(&m); - bindInfoObjects(&m); - bindCacheClient(&m); - bindVocabObjects(&m); - bindGraphData(&m); - bindDependIcuTokenizerOps(&m); - bindRandomTransformTensorOps(&m); -} -} // namespace dataset -} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt index 0d072d1d85..3273822db5 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/kernels/CMakeLists.txt @@ -4,16 +4,16 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) if (ENABLE_PYTHON) add_library(kernels OBJECT - compose_op.cc - random_apply_op.cc - random_choice_op.cc + data/compose_op.cc + data/random_apply_op.cc + data/random_choice_op.cc py_func_op.cc tensor_op.cc) target_include_directories(kernels PRIVATE ${pybind11_INCLUDE_DIRS}) else() add_library(kernels OBJECT - compose_op.cc - random_apply_op.cc - random_choice_op.cc + data/compose_op.cc + data/random_apply_op.cc + data/random_choice_op.cc tensor_op.cc) endif() diff --git a/mindspore/ccsrc/minddata/dataset/kernels/compose_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/data/compose_op.cc similarity index 97% rename from mindspore/ccsrc/minddata/dataset/kernels/compose_op.cc rename to mindspore/ccsrc/minddata/dataset/kernels/data/compose_op.cc index 152d779f1e..b50e0ae24b 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/compose_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/compose_op.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "minddata/dataset/kernels/compose_op.h" +#include "minddata/dataset/kernels/data/compose_op.h" #include diff --git a/mindspore/ccsrc/minddata/dataset/kernels/compose_op.h b/mindspore/ccsrc/minddata/dataset/kernels/data/compose_op.h similarity index 100% rename from mindspore/ccsrc/minddata/dataset/kernels/compose_op.h rename to mindspore/ccsrc/minddata/dataset/kernels/data/compose_op.h diff --git a/mindspore/ccsrc/minddata/dataset/kernels/no_op.h b/mindspore/ccsrc/minddata/dataset/kernels/data/no_op.h similarity index 100% rename from mindspore/ccsrc/minddata/dataset/kernels/no_op.h rename to mindspore/ccsrc/minddata/dataset/kernels/data/no_op.h diff --git a/mindspore/ccsrc/minddata/dataset/kernels/random_apply_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.cc similarity index 97% rename from mindspore/ccsrc/minddata/dataset/kernels/random_apply_op.cc rename to mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.cc index 40bd551013..9fe1d875e2 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/random_apply_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "minddata/dataset/kernels/random_apply_op.h" +#include "minddata/dataset/kernels/data/random_apply_op.h" #include #include diff --git a/mindspore/ccsrc/minddata/dataset/kernels/random_apply_op.h b/mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.h similarity index 98% rename from mindspore/ccsrc/minddata/dataset/kernels/random_apply_op.h rename to mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.h index 0d6b49dac3..8edc3e1d70 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/random_apply_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/random_apply_op.h @@ -24,7 +24,7 @@ #include #include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/kernels/compose_op.h" +#include "minddata/dataset/kernels/data/compose_op.h" #include "minddata/dataset/kernels/tensor_op.h" #include "minddata/dataset/util/random.h" diff --git a/mindspore/ccsrc/minddata/dataset/kernels/random_choice_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.cc similarity index 98% rename from mindspore/ccsrc/minddata/dataset/kernels/random_choice_op.cc rename to mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.cc index e54278fdd3..ee505b0dc2 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/random_choice_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "minddata/dataset/kernels/random_choice_op.h" +#include "minddata/dataset/kernels/data/random_choice_op.h" #include #include diff --git a/mindspore/ccsrc/minddata/dataset/kernels/random_choice_op.h b/mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.h similarity index 98% rename from mindspore/ccsrc/minddata/dataset/kernels/random_choice_op.h rename to mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.h index b7efb1316d..7244ca8931 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/random_choice_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/random_choice_op.h @@ -25,7 +25,7 @@ #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/kernels/tensor_op.h" -#include "minddata/dataset/kernels/compose_op.h" +#include "minddata/dataset/kernels/data/compose_op.h" #include "minddata/dataset/util/random.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h b/mindspore/ccsrc/minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h index d3e7871aec..f46101cc48 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h @@ -29,7 +29,6 @@ namespace mindspore { namespace dataset { class RandomVerticalFlipWithBBoxOp : public TensorOp { public: - // Default values, also used by python_bindings.cc static const float kDefProbability; // Constructor for RandomVerticalFlipWithBBoxOp // @param probability: Probablity of Image flipping, 0.5 by default