From 1f11ccce06c71bd5f79a9b86d5593a72af21bbca Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 14 Oct 2020 22:26:33 -0400 Subject: [PATCH] Removed log_adapter dependency for minddata lite Brute force fix? Added missing import Added removal of new cache functionality Fix batch Remove crc and cache optimizer Remove affine Remove cut mix batch Added securec.h Removed unused crc import Fixed minor arm64 compile issues for tensor ops Removed excess code from de_tensor Added vision import remove random device ops Removed commented line Added extra layer to switch adapter logic Added securec Added float fix --- .../ccsrc/minddata/dataset/api/de_tensor.cc | 20 +++------------ .../dataset/api/python/de_pipeline.cc | 2 +- .../minddata/dataset/core/config_manager.cc | 6 ++++- .../ccsrc/minddata/dataset/core/data_type.cc | 4 +++ .../ccsrc/minddata/dataset/core/tensor.h | 6 ++++- .../minddata/dataset/core/tensor_shape.cc | 4 +++ .../dataset/engine/cache/cache_grpc_server.cc | 6 ++++- .../minddata/dataset/engine/data_schema.cc | 4 +++ .../dataset/engine/datasetops/barrier_op.cc | 1 - .../engine/datasetops/cache_lookup_op.cc | 4 +++ .../dataset/engine/datasetops/cache_op.cc | 4 +++ .../dataset/engine/datasetops/dataset_op.cc | 6 +++++ .../dataset/engine/datasetops/dataset_op.h | 2 ++ .../engine/datasetops/epoch_ctrl_op.cc | 4 +++ .../dataset/engine/datasetops/filter_op.cc | 4 +++ .../engine/datasetops/map_op/map_op.cc | 4 +++ .../dataset/engine/datasetops/project_op.cc | 4 +++ .../dataset/engine/datasetops/rename_op.cc | 4 +++ .../dataset/engine/datasetops/repeat_op.cc | 4 +++ .../dataset/engine/datasetops/shuffle_op.cc | 4 +++ .../dataset/engine/datasetops/skip_op.cc | 4 +++ .../engine/datasetops/source/mindrecord_op.cc | 4 +++ .../dataset/engine/datasetops/zip_op.cc | 4 +++ .../minddata/dataset/engine/execution_tree.cc | 4 ++- .../dataset/engine/gnn/graph_shared_memory.cc | 4 +++ .../dataset/engine/gnn/grpc_async_server.cc | 4 +++ .../minddata/dataset/engine/perf/profiling.cc | 4 +++ .../minddata/dataset/engine/tdt/tdt_plugin.cc | 4 +++ .../ccsrc/minddata/dataset/include/tensor.h | 4 +++ .../dataset/kernels/image/affine_op.cc | 13 +++++----- .../dataset/kernels/image/cutmix_batch_op.cc | 2 +- .../ccsrc/minddata/dataset/text/vocab.cc | 4 +++ .../ccsrc/minddata/dataset/util/arena.cc | 2 +- .../minddata/dataset/util/btree_iterator.tpp | 3 ++- .../ccsrc/minddata/dataset/util/buddy.cc | 3 ++- .../minddata/dataset/util/circular_pool.cc | 2 +- .../minddata/dataset/util/data_helper.cc | 4 +-- .../ccsrc/minddata/dataset/util/data_helper.h | 2 +- .../minddata/dataset/util/intrp_service.h | 3 +-- mindspore/ccsrc/minddata/dataset/util/list.h | 2 +- mindspore/ccsrc/minddata/dataset/util/lock.cc | 2 +- .../ccsrc/minddata/dataset/util/log_adapter.h | 25 +++++++++++++++++++ mindspore/ccsrc/minddata/dataset/util/path.cc | 3 ++- mindspore/ccsrc/minddata/dataset/util/queue.h | 3 ++- .../ccsrc/minddata/dataset/util/random.h | 2 +- .../ccsrc/minddata/dataset/util/services.cc | 1 + .../dataset/util/storage_container.cc | 2 +- .../minddata/dataset/util/storage_manager.cc | 2 +- mindspore/ccsrc/minddata/dataset/util/task.cc | 2 +- mindspore/ccsrc/minddata/dataset/util/task.h | 2 +- mindspore/lite/minddata/CMakeLists.txt | 15 ++++++++--- .../lite/test/ut/src/dataset/eager_test.cc | 2 ++ 52 files changed, 182 insertions(+), 52 deletions(-) create mode 100644 mindspore/ccsrc/minddata/dataset/util/log_adapter.h diff --git a/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc b/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc index 190cc4efe9..0db2f1fcec 100644 --- a/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/api/de_tensor.cc @@ -14,14 +14,13 @@ * limitations under the License. */ -#include "minddata/dataset/include/de_tensor.h" -#include "minddata/dataset/include/type_id.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/data_type.h" +#include "minddata/dataset/include/de_tensor.h" +#include "minddata/dataset/include/type_id.h" #include "mindspore/core/ir/dtype/type_id.h" -#include "utils/hashing.h" #include "mindspore/lite/internal/include/ms_tensor.h" -#include "mindspore/core/utils/convert_utils_base.h" +#include "utils/hashing.h" namespace mindspore { namespace tensor { @@ -124,19 +123,6 @@ int DETensor::ElementsNum() const { return this->tensor_impl_->Size(); } -std::size_t DETensor::hash() const { - MS_ASSERT(this->tensor_impl_ != nullptr); - auto shape = this->shape(); - std::size_t hash_value = std::hash{}(SizeToInt(this->data_type())); - hash_value = hash_combine(hash_value, std::hash{}(shape.size())); - // hash all elements may costly, so only take at most 4 elements into account based on - // some experiments. - for (size_t i = 0; (i < shape.size()) && (i < 4); ++i) { - hash_value = hash_combine(hash_value, (std::hash{}(shape[i]))); - } - return hash_value; -} - size_t DETensor::Size() const { MS_ASSERT(this->tensor_impl_ != nullptr); return this->tensor_impl_->SizeInBytes(); diff --git a/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc index 68cba69db3..45d65288a2 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc @@ -41,6 +41,7 @@ #include "minddata/dataset/engine/datasetops/source/text_file_op.h" #include "minddata/dataset/engine/datasetops/source/voc_op.h" #include "minddata/dataset/kernels/py_func_op.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" #include "minddata/mindrecord/include/shard_category.h" @@ -51,7 +52,6 @@ #include "minddata/mindrecord/include/shard_shuffle.h" #include "minddata/mindrecord/include/shard_writer.h" #include "pybind11/stl.h" -#include "utils/log_adapter.h" #include "utils/ms_utils.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/core/config_manager.cc b/mindspore/ccsrc/minddata/dataset/core/config_manager.cc index 44901b8119..aa03200cdc 100644 --- a/mindspore/ccsrc/minddata/dataset/core/config_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/core/config_manager.cc @@ -20,7 +20,11 @@ #include #include -#include "mindspore/core/utils/log_adapter.h" +#ifndef ENABLE_ANDROID +#include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/util/system_pool.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/core/data_type.cc b/mindspore/ccsrc/minddata/dataset/core/data_type.cc index b5641e3105..77cf2f45a5 100644 --- a/mindspore/ccsrc/minddata/dataset/core/data_type.cc +++ b/mindspore/ccsrc/minddata/dataset/core/data_type.cc @@ -18,7 +18,11 @@ #include "minddata/dataset/core/pybind_support.h" #endif +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index 27b1a815fb..349f45a8f3 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -21,7 +21,11 @@ #include #include #include "./securec.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #if defined(_WIN32) || defined(_WIN64) #undef HAVE_STDDEF_H #undef HAVE_STDLIB_H @@ -33,12 +37,12 @@ #include "pybind11/stl.h" #endif -#include "utils/ms_utils.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor_helpers.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/util/status.h" +#include "utils/ms_utils.h" #ifndef ENABLE_ANDROID #include "proto/example.pb.h" #else diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc index 51fa874ea7..6a0fd22c33 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc @@ -20,7 +20,11 @@ #include #include "utils/ms_utils.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/core/constants.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc index 485d1a3806..892880f0ea 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "minddata/dataset/engine/cache/cache_grpc_server.h" #include +#include "minddata/dataset/engine/cache/cache_grpc_server.h" #include "minddata/dataset/engine/cache/cache_server.h" #include "minddata/dataset/util/path.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { CacheServerGreeterImpl::CacheServerGreeterImpl(int32_t port, int32_t shared_memory_sz_in_gb) diff --git a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc index 6db8ec5614..bffb87f0c6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc @@ -26,7 +26,11 @@ #include "utils/ms_utils.h" #include "minddata/dataset/util/status.h" #include "minddata/dataset/core/tensor_shape.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc index f77d39e3e5..f240c9bc4c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc @@ -21,7 +21,6 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc index 0e38ad6333..1cdaec76f8 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc @@ -19,8 +19,12 @@ #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/engine/execution_tree.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" #include "utils/system/crc32c.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc index 652d78bfbc..04abeb989e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc @@ -26,7 +26,11 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/util/task_manager.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index a24b4395a9..9e59236d01 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -30,8 +30,12 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/system/crc32c.h" #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { @@ -385,6 +389,7 @@ Status DatasetOp::FetchRemoveSampler(std::shared_ptr *sampler) { return Status::OK(); } +#ifndef ENABLE_ANDROID uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { std::stringstream ss; op->tree_->Print(ss, op); @@ -431,6 +436,7 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length()); return cache_crc; } +#endif void DatasetOp::UpdateRepeatAndEpochCounter() { op_current_repeats_++; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h index f99c0603d6..00e2200f4c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h @@ -338,8 +338,10 @@ class DatasetOp : public std::enable_shared_from_this { /// \return Status error code Status FetchRemoveSampler(std::shared_ptr *sampler); +#ifndef ENABLE_ANDROID // Computes a CRC value for the operator static uint32_t GenerateCRC(const std::shared_ptr &op); +#endif /// \brief A helper templated function for casting "this" pointer to shared_ptr /// Similar to shared_from_this, except this one will give you the derived class as shared_ptr diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc index 1343fd4608..b8c94a01c2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc @@ -22,7 +22,11 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc index 446cd80299..d9eb064041 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc @@ -29,7 +29,11 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/kernels/tensor_op.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/util/task_manager.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc index f8c628647f..0fa1efec4b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc @@ -31,7 +31,11 @@ #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/kernels/tensor_op.h" #include "minddata/dataset/util/task_manager.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc index 01fcab877c..b88578b32f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc @@ -26,7 +26,11 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc index deb4027436..b3f73b37b7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc @@ -25,7 +25,11 @@ #include "minddata/dataset/engine/data_buffer.h" #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc index 0a8591f5b1..557698a396 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc @@ -23,7 +23,11 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc index 282f57b8a9..1259cfcb1c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc @@ -34,7 +34,11 @@ #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc index 012e23b46a..c85af73e2c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc @@ -24,7 +24,11 @@ #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc index 6ba6a132f4..f9ddfb68de 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc @@ -30,7 +30,11 @@ #include "minddata/dataset/engine/db_connector.h" #include "minddata/dataset/engine/execution_tree.h" #include "minddata/dataset/engine/opt/pass.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc index 3b1fec02e0..0ab8406153 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc @@ -22,7 +22,11 @@ #include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc index 8a43574997..42660b4599 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc @@ -25,8 +25,8 @@ #ifndef ENABLE_ANDROID #include "minddata/dataset/engine/opt/pre/cache_transform_pass.h" #include "minddata/dataset/engine/opt/post/repeat_pass.h" -#endif #include "minddata/dataset/engine/opt/pre/cache_error_pass.h" +#endif #include "minddata/dataset/engine/opt/pre/epoch_injection_pass.h" #include "mindspore/ccsrc/minddata/dataset/engine/opt/optional/tensor_op_fusion_pass.h" #include "minddata/dataset/engine/perf/profiling.h" @@ -236,7 +236,9 @@ Status ExecutionTree::PrepareTreePreAction() { std::vector> pre_actions; // Construct pre actions MS_LOG(INFO) << "Running pre pass loops."; +#ifndef ENABLE_ANDROID pre_actions.push_back(std::make_unique()); +#endif pre_actions.push_back(std::make_unique()); pre_actions.push_back(std::make_unique()); #ifndef ENABLE_ANDROID diff --git a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc index 54e6eda0d2..019f1513fa 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_shared_memory.cc @@ -18,7 +18,11 @@ #include +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc b/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc index 02c967e442..2202f2f75b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/gnn/grpc_async_server.cc @@ -18,7 +18,11 @@ #include #include "minddata/dataset/util/task_manager.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc b/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc index 82a4b6818c..f3d4a39acb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/perf/profiling.cc @@ -24,7 +24,11 @@ #include "minddata/dataset/engine/perf/connector_size.h" #include "minddata/dataset/engine/perf/connector_throughput.h" #include "minddata/dataset/engine/perf/dataset_iterator_tracing.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc index dd57fa7ea4..5f4834c80f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc @@ -15,7 +15,11 @@ */ #include "minddata/dataset/engine/tdt/tdt_plugin.h" #include "utils/ms_utils.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #include "minddata/dataset/engine/perf/profiling.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/include/tensor.h b/mindspore/ccsrc/minddata/dataset/include/tensor.h index b2fe352c1d..6d43f18d31 100644 --- a/mindspore/ccsrc/minddata/dataset/include/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/include/tensor.h @@ -21,7 +21,11 @@ #include #include #include "./securec.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif #if defined(_WIN32) || defined(_WIN64) #undef HAVE_STDDEF_H #undef HAVE_STDLIB_H diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc index 1ea1ea434c..90403abfc9 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc @@ -74,12 +74,13 @@ Status AffineOp::Compute(const std::shared_ptr &input, std::shared_ptrmat().cols - 1) / 2.0); float_t cy = ((input_cv->mat().rows - 1) / 2.0); // Calculate RSS - std::vector matrix{scale_ * cos(degrees + shear_y) / cos(shear_y), - scale_ * (-1 * cos(degrees + shear_y) * tan(shear_x) / cos(shear_y) - sin(degrees)), - 0, - scale_ * sin(degrees + shear_y) / cos(shear_y), - scale_ * (-1 * sin(degrees + shear_y) * tan(shear_x) / cos(shear_y) + cos(degrees)), - 0}; + std::vector matrix{ + static_cast(scale_ * cos(degrees + shear_y) / cos(shear_y)), + static_cast(scale_ * (-1 * cos(degrees + shear_y) * tan(shear_x) / cos(shear_y) - sin(degrees))), + 0, + static_cast(scale_ * sin(degrees + shear_y) / cos(shear_y)), + static_cast(scale_ * (-1 * sin(degrees + shear_y) * tan(shear_x) / cos(shear_y) + cos(degrees))), + 0}; // Compute T * C * RSS * C^-1 matrix[2] = (1 - matrix[0]) * cx - matrix[1] * cy + translation_x; matrix[5] = (1 - matrix[4]) * cy - matrix[3] * cx + translation_y; diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc index d416654599..6480faa41f 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc @@ -182,7 +182,7 @@ Status CutMixBatchOp::Compute(const TensorRow &input, TensorRow *output) { void CutMixBatchOp::Print(std::ostream &out) const { out << "CutMixBatchOp: " - << "image_batch_format: " << image_batch_format_ << "alpha: " << alpha_ << ", probability: " << prob_ << "\n"; + << "\n"; } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/text/vocab.cc b/mindspore/ccsrc/minddata/dataset/text/vocab.cc index 3ef1c399dd..b0f7349a71 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/vocab.cc @@ -20,7 +20,11 @@ #include #include "minddata/dataset/text/vocab.h" +#ifndef ENABLE_ANDROID #include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/arena.cc b/mindspore/ccsrc/minddata/dataset/util/arena.cc index 6bd6d295ff..de563cf1db 100644 --- a/mindspore/ccsrc/minddata/dataset/util/arena.cc +++ b/mindspore/ccsrc/minddata/dataset/util/arena.cc @@ -16,9 +16,9 @@ #include "minddata/dataset/util/arena.h" #include #include +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/system_pool.h" #include "./securec.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp b/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp index 3e72013911..890b189096 100644 --- a/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp +++ b/mindspore/ccsrc/minddata/dataset/util/btree_iterator.tpp @@ -15,7 +15,8 @@ #ifndef DATASET_UTIL_BTREE_ITERATOR_H_ #define DATASET_UTIL_BTREE_ITERATOR_H_ -#include "utils/log_adapter.h" +#include "./securec.h" +#include "minddata/dataset/util/log_adapter.h" #include "btree.h" namespace mindspore { diff --git a/mindspore/ccsrc/minddata/dataset/util/buddy.cc b/mindspore/ccsrc/minddata/dataset/util/buddy.cc index 0cc2baed5f..bbeeaff0db 100644 --- a/mindspore/ccsrc/minddata/dataset/util/buddy.cc +++ b/mindspore/ccsrc/minddata/dataset/util/buddy.cc @@ -16,9 +16,10 @@ #include "minddata/dataset/util/buddy.h" #include #include + #include "minddata/dataset/util/memory_pool.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/system_pool.h" -#include "utils/log_adapter.h" #include "./securec.h" inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } diff --git a/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc b/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc index 5f57d596eb..539c39cc77 100644 --- a/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc +++ b/mindspore/ccsrc/minddata/dataset/util/circular_pool.cc @@ -19,8 +19,8 @@ #include #include #include "./securec.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/system_pool.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/data_helper.cc b/mindspore/ccsrc/minddata/dataset/util/data_helper.cc index ac821c2214..49c70f884b 100644 --- a/mindspore/ccsrc/minddata/dataset/util/data_helper.cc +++ b/mindspore/ccsrc/minddata/dataset/util/data_helper.cc @@ -23,11 +23,11 @@ #include #include -#include "minddata/dataset/util/status.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" +#include "minddata/dataset/util/status.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/data_helper.h b/mindspore/ccsrc/minddata/dataset/util/data_helper.h index 19a0195958..bf2e7f10a4 100644 --- a/mindspore/ccsrc/minddata/dataset/util/data_helper.h +++ b/mindspore/ccsrc/minddata/dataset/util/data_helper.h @@ -30,7 +30,7 @@ #include "minddata/dataset/core/data_type.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/status.h" diff --git a/mindspore/ccsrc/minddata/dataset/util/intrp_service.h b/mindspore/ccsrc/minddata/dataset/util/intrp_service.h index 2aa6987e6b..13c8b43673 100644 --- a/mindspore/ccsrc/minddata/dataset/util/intrp_service.h +++ b/mindspore/ccsrc/minddata/dataset/util/intrp_service.h @@ -23,12 +23,11 @@ #include #include "minddata/dataset/util/allocator.h" #include "minddata/dataset/util/intrp_resource.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/service.h" #include "minddata/dataset/util/services.h" #include "minddata/dataset/util/status.h" -#include "utils/log_adapter.h" - namespace mindspore { namespace dataset { using SvcAllocator = Allocator>; diff --git a/mindspore/ccsrc/minddata/dataset/util/list.h b/mindspore/ccsrc/minddata/dataset/util/list.h index b7eb8ebd01..06c1d069f5 100644 --- a/mindspore/ccsrc/minddata/dataset/util/list.h +++ b/mindspore/ccsrc/minddata/dataset/util/list.h @@ -19,7 +19,7 @@ #include #include -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/lock.cc b/mindspore/ccsrc/minddata/dataset/util/lock.cc index 5302196a46..fe4a9c2192 100644 --- a/mindspore/ccsrc/minddata/dataset/util/lock.cc +++ b/mindspore/ccsrc/minddata/dataset/util/lock.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "minddata/dataset/util/lock.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/log_adapter.h b/mindspore/ccsrc/minddata/dataset/util/log_adapter.h new file mode 100644 index 0000000000..38dfafd3dc --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/util/log_adapter.h @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ + +#ifndef ENABLE_ANDROID +#include "utils/log_adapter.h" +#else +#include "mindspore/lite/src/common/log_adapter.h" +#endif + +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ diff --git a/mindspore/ccsrc/minddata/dataset/util/path.cc b/mindspore/ccsrc/minddata/dataset/util/path.cc index 97416be6a8..df61ccda27 100644 --- a/mindspore/ccsrc/minddata/dataset/util/path.cc +++ b/mindspore/ccsrc/minddata/dataset/util/path.cc @@ -22,8 +22,9 @@ #include #include +#include "./securec.h" #include "utils/ms_utils.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/queue.h b/mindspore/ccsrc/minddata/dataset/util/queue.h index 417a1ab507..2f2e0c7006 100644 --- a/mindspore/ccsrc/minddata/dataset/util/queue.h +++ b/mindspore/ccsrc/minddata/dataset/util/queue.h @@ -24,9 +24,10 @@ #include #include +#include "./securec.h" #include "utils/ms_utils.h" -#include "utils/log_adapter.h" #include "minddata/dataset/util/allocator.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/services.h" #include "minddata/dataset/util/cond_var.h" #include "minddata/dataset/util/task_manager.h" diff --git a/mindspore/ccsrc/minddata/dataset/util/random.h b/mindspore/ccsrc/minddata/dataset/util/random.h index 7e8dd48e88..4d4e0b0d32 100644 --- a/mindspore/ccsrc/minddata/dataset/util/random.h +++ b/mindspore/ccsrc/minddata/dataset/util/random.h @@ -28,7 +28,7 @@ #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" -#include "utils/log_adapter.h" +#include "minddata/dataset/util/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/services.cc b/mindspore/ccsrc/minddata/dataset/util/services.cc index 7958aad224..c6e0f38d24 100644 --- a/mindspore/ccsrc/minddata/dataset/util/services.cc +++ b/mindspore/ccsrc/minddata/dataset/util/services.cc @@ -22,6 +22,7 @@ #include #endif #include +#include "./securec.h" #include "minddata/dataset/util/circular_pool.h" #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/task_manager.h" diff --git a/mindspore/ccsrc/minddata/dataset/util/storage_container.cc b/mindspore/ccsrc/minddata/dataset/util/storage_container.cc index b0855512cd..cca10b40c4 100644 --- a/mindspore/ccsrc/minddata/dataset/util/storage_container.cc +++ b/mindspore/ccsrc/minddata/dataset/util/storage_container.cc @@ -20,9 +20,9 @@ #include #include #include "utils/ms_utils.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/status.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc b/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc index 8c99e5a91e..bb70198fd1 100644 --- a/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/util/storage_manager.cc @@ -20,9 +20,9 @@ #include #include #include "utils/ms_utils.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/util/services.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/util/task.cc b/mindspore/ccsrc/minddata/dataset/util/task.cc index 9cbaa7074b..77336e56ae 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task.cc +++ b/mindspore/ccsrc/minddata/dataset/util/task.cc @@ -15,8 +15,8 @@ */ #include "minddata/dataset/util/task.h" #include "utils/ms_utils.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/task_manager.h" -#include "utils/log_adapter.h" #if defined(__ANDROID__) || defined(ANDROID) #include "minddata/dataset/util/services.h" #endif diff --git a/mindspore/ccsrc/minddata/dataset/util/task.h b/mindspore/ccsrc/minddata/dataset/util/task.h index 951beb9a4c..b41861ecc0 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task.h +++ b/mindspore/ccsrc/minddata/dataset/util/task.h @@ -29,10 +29,10 @@ #include #include "minddata/dataset/util/intrp_resource.h" #include "minddata/dataset/util/list.h" +#include "minddata/dataset/util/log_adapter.h" #include "minddata/dataset/util/memory_pool.h" #include "minddata/dataset/util/services.h" #include "minddata/dataset/util/wait_post.h" -#include "utils/log_adapter.h" namespace mindspore { namespace dataset { diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index fe767f665e..dbc0d1c158 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -118,8 +118,15 @@ if (BUILD_MINDDATA STREQUAL "full") list(REMOVE_ITEM MINDDATA_ENGINE_OPT_PRE_SRC_FILES "${MINDDATA_DIR}/engine/opt/pre/cache_transform_pass.cc" + "${MINDDATA_DIR}/engine/opt/pre/cache_error_pass.cc" ) + #list(REMOVE_ITEM MINDDATA_KERNELS_IMAGE_SRC_FILES + #"${MINDDATA_DIR}/kernels/image/affine_op.cc" + #"${MINDDATA_DIR}/kernels/image/cutmix_batch_op.cc" + #"${MINDDATA_DIR}/kernels/image/random_affine_op.cc" + #) + include_directories("${CMAKE_BINARY_DIR}/minddata/dataset/engine/cache") if (BUILD_MINDDATA_EXAMPLE AND (PLATFORM_ARM32 OR PLATFORM_ARM64)) @@ -149,8 +156,6 @@ if (BUILD_MINDDATA STREQUAL "full") ${MINDDATA_EXAMPLE_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/log_adapter.cc ${CORE_DIR}/utils/ms_utils.cc - ${CORE_DIR}/gvar/logging_level.cc - ${CCSRC_DIR}/utils/system/crc32c.cc ) target_link_libraries(minddata-lite @@ -189,6 +194,10 @@ elseif (BUILD_MINDDATA STREQUAL "lite") ${MINDDATA_DIR}/api/execute.cc ) + list(REMOVE_ITEM MINDDATA_KERNELS_IMAGE_SRC_FILES + "${MINDDATA_DIR}/kernels/image/bounding_box_augment_op.cc" + "${MINDDATA_DIR}/kernels/image/cut_out_op.cc" + ) add_library(minddata-lite SHARED ${MINDDATA_CORE_SRC_FILES} ${MINDDATA_KERNELS_SRC_FILES} @@ -198,8 +207,6 @@ elseif (BUILD_MINDDATA STREQUAL "lite") ${MINDDATA_DIR}/util/memory_pool.cc ${MINDDATA_DIR}/util/path.cc ${MINDDATA_DIR}/api/transforms.cc - ${CORE_DIR}/utils/log_adapter.cc - ${CORE_DIR}/gvar/logging_level.cc ) target_link_libraries(minddata-lite diff --git a/mindspore/lite/test/ut/src/dataset/eager_test.cc b/mindspore/lite/test/ut/src/dataset/eager_test.cc index cf3d02a040..04412678f2 100644 --- a/mindspore/lite/test/ut/src/dataset/eager_test.cc +++ b/mindspore/lite/test/ut/src/dataset/eager_test.cc @@ -19,6 +19,8 @@ #include "./securec.h" #include "minddata/dataset/include/tensor.h" #include "minddata/dataset/include/datasets.h" +#include "minddata/dataset/include/transforms.h" +#include "minddata/dataset/include/vision.h" #include "minddata/dataset/include/execute.h" #include "minddata/dataset/util/path.h"