Brute force fix? Added missing import Added removal of new cache functionality Fix batch Remove crc and cache optimizer Remove affine Remove cut mix batch Added securec.h Removed unused crc import Fixed minor arm64 compile issues for tensor ops Removed excess code from de_tensor Added vision import remove random device ops Removed commented line Added extra layer to switch adapter logic Added securec Added float fixtags/v1.1.0
| @@ -14,14 +14,13 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #include "minddata/dataset/include/de_tensor.h" | |||||
| #include "minddata/dataset/include/type_id.h" | |||||
| #include "minddata/dataset/core/constants.h" | #include "minddata/dataset/core/constants.h" | ||||
| #include "minddata/dataset/core/data_type.h" | #include "minddata/dataset/core/data_type.h" | ||||
| #include "minddata/dataset/include/de_tensor.h" | |||||
| #include "minddata/dataset/include/type_id.h" | |||||
| #include "mindspore/core/ir/dtype/type_id.h" | #include "mindspore/core/ir/dtype/type_id.h" | ||||
| #include "utils/hashing.h" | |||||
| #include "mindspore/lite/internal/include/ms_tensor.h" | #include "mindspore/lite/internal/include/ms_tensor.h" | ||||
| #include "mindspore/core/utils/convert_utils_base.h" | |||||
| #include "utils/hashing.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace tensor { | namespace tensor { | ||||
| @@ -124,19 +123,6 @@ int DETensor::ElementsNum() const { | |||||
| return this->tensor_impl_->Size(); | return this->tensor_impl_->Size(); | ||||
| } | } | ||||
| std::size_t DETensor::hash() const { | |||||
| MS_ASSERT(this->tensor_impl_ != nullptr); | |||||
| auto shape = this->shape(); | |||||
| std::size_t hash_value = std::hash<int>{}(SizeToInt(this->data_type())); | |||||
| hash_value = hash_combine(hash_value, std::hash<size_t>{}(shape.size())); | |||||
| // hash all elements may costly, so only take at most 4 elements into account based on | |||||
| // some experiments. | |||||
| for (size_t i = 0; (i < shape.size()) && (i < 4); ++i) { | |||||
| hash_value = hash_combine(hash_value, (std::hash<int>{}(shape[i]))); | |||||
| } | |||||
| return hash_value; | |||||
| } | |||||
| size_t DETensor::Size() const { | size_t DETensor::Size() const { | ||||
| MS_ASSERT(this->tensor_impl_ != nullptr); | MS_ASSERT(this->tensor_impl_ != nullptr); | ||||
| return this->tensor_impl_->SizeInBytes(); | return this->tensor_impl_->SizeInBytes(); | ||||
| @@ -41,6 +41,7 @@ | |||||
| #include "minddata/dataset/engine/datasetops/source/text_file_op.h" | #include "minddata/dataset/engine/datasetops/source/text_file_op.h" | ||||
| #include "minddata/dataset/engine/datasetops/source/voc_op.h" | #include "minddata/dataset/engine/datasetops/source/voc_op.h" | ||||
| #include "minddata/dataset/kernels/py_func_op.h" | #include "minddata/dataset/kernels/py_func_op.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/random.h" | #include "minddata/dataset/util/random.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| #include "minddata/mindrecord/include/shard_category.h" | #include "minddata/mindrecord/include/shard_category.h" | ||||
| @@ -51,7 +52,6 @@ | |||||
| #include "minddata/mindrecord/include/shard_shuffle.h" | #include "minddata/mindrecord/include/shard_shuffle.h" | ||||
| #include "minddata/mindrecord/include/shard_writer.h" | #include "minddata/mindrecord/include/shard_writer.h" | ||||
| #include "pybind11/stl.h" | #include "pybind11/stl.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -20,7 +20,11 @@ | |||||
| #include <string> | #include <string> | ||||
| #include <utility> | #include <utility> | ||||
| #include "mindspore/core/utils/log_adapter.h" | |||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | |||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #include "minddata/dataset/util/system_pool.h" | #include "minddata/dataset/util/system_pool.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -18,7 +18,11 @@ | |||||
| #include "minddata/dataset/core/pybind_support.h" | #include "minddata/dataset/core/pybind_support.h" | ||||
| #endif | #endif | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -21,7 +21,11 @@ | |||||
| #include <string> | #include <string> | ||||
| #include <vector> | #include <vector> | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #if defined(_WIN32) || defined(_WIN64) | #if defined(_WIN32) || defined(_WIN64) | ||||
| #undef HAVE_STDDEF_H | #undef HAVE_STDDEF_H | ||||
| #undef HAVE_STDLIB_H | #undef HAVE_STDLIB_H | ||||
| @@ -33,12 +37,12 @@ | |||||
| #include "pybind11/stl.h" | #include "pybind11/stl.h" | ||||
| #endif | #endif | ||||
| #include "utils/ms_utils.h" | |||||
| #include "minddata/dataset/core/constants.h" | #include "minddata/dataset/core/constants.h" | ||||
| #include "minddata/dataset/core/data_type.h" | #include "minddata/dataset/core/data_type.h" | ||||
| #include "minddata/dataset/core/tensor_helpers.h" | #include "minddata/dataset/core/tensor_helpers.h" | ||||
| #include "minddata/dataset/core/tensor_shape.h" | #include "minddata/dataset/core/tensor_shape.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| #include "utils/ms_utils.h" | |||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| #include "proto/example.pb.h" | #include "proto/example.pb.h" | ||||
| #else | #else | ||||
| @@ -20,7 +20,11 @@ | |||||
| #include <limits> | #include <limits> | ||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #include "minddata/dataset/core/constants.h" | #include "minddata/dataset/core/constants.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -13,11 +13,15 @@ | |||||
| * See the License for the specific language governing permissions and | * See the License for the specific language governing permissions and | ||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #include "minddata/dataset/engine/cache/cache_grpc_server.h" | |||||
| #include <limits> | #include <limits> | ||||
| #include "minddata/dataset/engine/cache/cache_grpc_server.h" | |||||
| #include "minddata/dataset/engine/cache/cache_server.h" | #include "minddata/dataset/engine/cache/cache_server.h" | ||||
| #include "minddata/dataset/util/path.h" | #include "minddata/dataset/util/path.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| CacheServerGreeterImpl::CacheServerGreeterImpl(int32_t port, int32_t shared_memory_sz_in_gb) | CacheServerGreeterImpl::CacheServerGreeterImpl(int32_t port, int32_t shared_memory_sz_in_gb) | ||||
| @@ -26,7 +26,11 @@ | |||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| #include "minddata/dataset/core/tensor_shape.h" | #include "minddata/dataset/core/tensor_shape.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -21,7 +21,6 @@ | |||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/core/config_manager.h" | #include "minddata/dataset/core/config_manager.h" | ||||
| #include "minddata/dataset/core/global_context.h" | #include "minddata/dataset/core/global_context.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -19,8 +19,12 @@ | |||||
| #include "minddata/dataset/core/constants.h" | #include "minddata/dataset/core/constants.h" | ||||
| #include "minddata/dataset/core/global_context.h" | #include "minddata/dataset/core/global_context.h" | ||||
| #include "minddata/dataset/engine/execution_tree.h" | #include "minddata/dataset/engine/execution_tree.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #include "utils/system/crc32c.h" | #include "utils/system/crc32c.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -26,7 +26,11 @@ | |||||
| #include "minddata/dataset/engine/execution_tree.h" | #include "minddata/dataset/engine/execution_tree.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -30,8 +30,12 @@ | |||||
| #include "minddata/dataset/engine/data_buffer.h" | #include "minddata/dataset/engine/data_buffer.h" | ||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/system/crc32c.h" | #include "utils/system/crc32c.h" | ||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -385,6 +389,7 @@ Status DatasetOp::FetchRemoveSampler(std::shared_ptr<Sampler> *sampler) { | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| #ifndef ENABLE_ANDROID | |||||
| uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) { | uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) { | ||||
| std::stringstream ss; | std::stringstream ss; | ||||
| op->tree_->Print(ss, op); | op->tree_->Print(ss, op); | ||||
| @@ -431,6 +436,7 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) { | |||||
| uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length()); | uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length()); | ||||
| return cache_crc; | return cache_crc; | ||||
| } | } | ||||
| #endif | |||||
| void DatasetOp::UpdateRepeatAndEpochCounter() { | void DatasetOp::UpdateRepeatAndEpochCounter() { | ||||
| op_current_repeats_++; | op_current_repeats_++; | ||||
| @@ -338,8 +338,10 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> { | |||||
| /// \return Status error code | /// \return Status error code | ||||
| Status FetchRemoveSampler(std::shared_ptr<Sampler> *sampler); | Status FetchRemoveSampler(std::shared_ptr<Sampler> *sampler); | ||||
| #ifndef ENABLE_ANDROID | |||||
| // Computes a CRC value for the operator | // Computes a CRC value for the operator | ||||
| static uint32_t GenerateCRC(const std::shared_ptr<DatasetOp> &op); | static uint32_t GenerateCRC(const std::shared_ptr<DatasetOp> &op); | ||||
| #endif | |||||
| /// \brief A helper templated function for casting "this" pointer to shared_ptr<derived> | /// \brief A helper templated function for casting "this" pointer to shared_ptr<derived> | ||||
| /// Similar to shared_from_this, except this one will give you the derived class as shared_ptr | /// Similar to shared_from_this, except this one will give you the derived class as shared_ptr | ||||
| @@ -22,7 +22,11 @@ | |||||
| #include "minddata/dataset/engine/data_buffer.h" | #include "minddata/dataset/engine/data_buffer.h" | ||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -29,7 +29,11 @@ | |||||
| #include "minddata/dataset/engine/execution_tree.h" | #include "minddata/dataset/engine/execution_tree.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #include "minddata/dataset/kernels/tensor_op.h" | #include "minddata/dataset/kernels/tensor_op.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -31,7 +31,11 @@ | |||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #include "minddata/dataset/kernels/tensor_op.h" | #include "minddata/dataset/kernels/tensor_op.h" | ||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -26,7 +26,11 @@ | |||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/engine/execution_tree.h" | #include "minddata/dataset/engine/execution_tree.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -25,7 +25,11 @@ | |||||
| #include "minddata/dataset/engine/data_buffer.h" | #include "minddata/dataset/engine/data_buffer.h" | ||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -23,7 +23,11 @@ | |||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -34,7 +34,11 @@ | |||||
| #include "minddata/dataset/util/random.h" | #include "minddata/dataset/util/random.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -24,7 +24,11 @@ | |||||
| #include "minddata/dataset/engine/execution_tree.h" | #include "minddata/dataset/engine/execution_tree.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -30,7 +30,11 @@ | |||||
| #include "minddata/dataset/engine/db_connector.h" | #include "minddata/dataset/engine/db_connector.h" | ||||
| #include "minddata/dataset/engine/execution_tree.h" | #include "minddata/dataset/engine/execution_tree.h" | ||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -22,7 +22,11 @@ | |||||
| #include "minddata/dataset/engine/opt/pass.h" | #include "minddata/dataset/engine/opt/pass.h" | ||||
| #include "minddata/dataset/core/config_manager.h" | #include "minddata/dataset/core/config_manager.h" | ||||
| #include "minddata/dataset/core/global_context.h" | #include "minddata/dataset/core/global_context.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -25,8 +25,8 @@ | |||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| #include "minddata/dataset/engine/opt/pre/cache_transform_pass.h" | #include "minddata/dataset/engine/opt/pre/cache_transform_pass.h" | ||||
| #include "minddata/dataset/engine/opt/post/repeat_pass.h" | #include "minddata/dataset/engine/opt/post/repeat_pass.h" | ||||
| #endif | |||||
| #include "minddata/dataset/engine/opt/pre/cache_error_pass.h" | #include "minddata/dataset/engine/opt/pre/cache_error_pass.h" | ||||
| #endif | |||||
| #include "minddata/dataset/engine/opt/pre/epoch_injection_pass.h" | #include "minddata/dataset/engine/opt/pre/epoch_injection_pass.h" | ||||
| #include "mindspore/ccsrc/minddata/dataset/engine/opt/optional/tensor_op_fusion_pass.h" | #include "mindspore/ccsrc/minddata/dataset/engine/opt/optional/tensor_op_fusion_pass.h" | ||||
| #include "minddata/dataset/engine/perf/profiling.h" | #include "minddata/dataset/engine/perf/profiling.h" | ||||
| @@ -236,7 +236,9 @@ Status ExecutionTree::PrepareTreePreAction() { | |||||
| std::vector<std::unique_ptr<Pass>> pre_actions; | std::vector<std::unique_ptr<Pass>> pre_actions; | ||||
| // Construct pre actions | // Construct pre actions | ||||
| MS_LOG(INFO) << "Running pre pass loops."; | MS_LOG(INFO) << "Running pre pass loops."; | ||||
| #ifndef ENABLE_ANDROID | |||||
| pre_actions.push_back(std::make_unique<CacheErrorPass>()); | pre_actions.push_back(std::make_unique<CacheErrorPass>()); | ||||
| #endif | |||||
| pre_actions.push_back(std::make_unique<EpochInjectionPass>()); | pre_actions.push_back(std::make_unique<EpochInjectionPass>()); | ||||
| pre_actions.push_back(std::make_unique<RemovalPass>()); | pre_actions.push_back(std::make_unique<RemovalPass>()); | ||||
| #ifndef ENABLE_ANDROID | #ifndef ENABLE_ANDROID | ||||
| @@ -18,7 +18,11 @@ | |||||
| #include <string> | #include <string> | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -18,7 +18,11 @@ | |||||
| #include <limits> | #include <limits> | ||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -24,7 +24,11 @@ | |||||
| #include "minddata/dataset/engine/perf/connector_size.h" | #include "minddata/dataset/engine/perf/connector_size.h" | ||||
| #include "minddata/dataset/engine/perf/connector_throughput.h" | #include "minddata/dataset/engine/perf/connector_throughput.h" | ||||
| #include "minddata/dataset/engine/perf/dataset_iterator_tracing.h" | #include "minddata/dataset/engine/perf/dataset_iterator_tracing.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -15,7 +15,11 @@ | |||||
| */ | */ | ||||
| #include "minddata/dataset/engine/tdt/tdt_plugin.h" | #include "minddata/dataset/engine/tdt/tdt_plugin.h" | ||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #include "minddata/dataset/engine/perf/profiling.h" | #include "minddata/dataset/engine/perf/profiling.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -21,7 +21,11 @@ | |||||
| #include <string> | #include <string> | ||||
| #include <vector> | #include <vector> | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #if defined(_WIN32) || defined(_WIN64) | #if defined(_WIN32) || defined(_WIN64) | ||||
| #undef HAVE_STDDEF_H | #undef HAVE_STDDEF_H | ||||
| #undef HAVE_STDLIB_H | #undef HAVE_STDLIB_H | ||||
| @@ -74,12 +74,13 @@ Status AffineOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<T | |||||
| float_t cx = ((input_cv->mat().cols - 1) / 2.0); | float_t cx = ((input_cv->mat().cols - 1) / 2.0); | ||||
| float_t cy = ((input_cv->mat().rows - 1) / 2.0); | float_t cy = ((input_cv->mat().rows - 1) / 2.0); | ||||
| // Calculate RSS | // Calculate RSS | ||||
| std::vector<float_t> matrix{scale_ * cos(degrees + shear_y) / cos(shear_y), | |||||
| scale_ * (-1 * cos(degrees + shear_y) * tan(shear_x) / cos(shear_y) - sin(degrees)), | |||||
| 0, | |||||
| scale_ * sin(degrees + shear_y) / cos(shear_y), | |||||
| scale_ * (-1 * sin(degrees + shear_y) * tan(shear_x) / cos(shear_y) + cos(degrees)), | |||||
| 0}; | |||||
| std::vector<float_t> matrix{ | |||||
| static_cast<float>(scale_ * cos(degrees + shear_y) / cos(shear_y)), | |||||
| static_cast<float>(scale_ * (-1 * cos(degrees + shear_y) * tan(shear_x) / cos(shear_y) - sin(degrees))), | |||||
| 0, | |||||
| static_cast<float>(scale_ * sin(degrees + shear_y) / cos(shear_y)), | |||||
| static_cast<float>(scale_ * (-1 * sin(degrees + shear_y) * tan(shear_x) / cos(shear_y) + cos(degrees))), | |||||
| 0}; | |||||
| // Compute T * C * RSS * C^-1 | // Compute T * C * RSS * C^-1 | ||||
| matrix[2] = (1 - matrix[0]) * cx - matrix[1] * cy + translation_x; | matrix[2] = (1 - matrix[0]) * cx - matrix[1] * cy + translation_x; | ||||
| matrix[5] = (1 - matrix[4]) * cy - matrix[3] * cx + translation_y; | matrix[5] = (1 - matrix[4]) * cy - matrix[3] * cx + translation_y; | ||||
| @@ -182,7 +182,7 @@ Status CutMixBatchOp::Compute(const TensorRow &input, TensorRow *output) { | |||||
| void CutMixBatchOp::Print(std::ostream &out) const { | void CutMixBatchOp::Print(std::ostream &out) const { | ||||
| out << "CutMixBatchOp: " | out << "CutMixBatchOp: " | ||||
| << "image_batch_format: " << image_batch_format_ << "alpha: " << alpha_ << ", probability: " << prob_ << "\n"; | |||||
| << "\n"; | |||||
| } | } | ||||
| } // namespace dataset | } // namespace dataset | ||||
| } // namespace mindspore | } // namespace mindspore | ||||
| @@ -20,7 +20,11 @@ | |||||
| #include <algorithm> | #include <algorithm> | ||||
| #include "minddata/dataset/text/vocab.h" | #include "minddata/dataset/text/vocab.h" | ||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -16,9 +16,9 @@ | |||||
| #include "minddata/dataset/util/arena.h" | #include "minddata/dataset/util/arena.h" | ||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include <utility> | #include <utility> | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/system_pool.h" | #include "minddata/dataset/util/system_pool.h" | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -15,7 +15,8 @@ | |||||
| #ifndef DATASET_UTIL_BTREE_ITERATOR_H_ | #ifndef DATASET_UTIL_BTREE_ITERATOR_H_ | ||||
| #define DATASET_UTIL_BTREE_ITERATOR_H_ | #define DATASET_UTIL_BTREE_ITERATOR_H_ | ||||
| #include "utils/log_adapter.h" | |||||
| #include "./securec.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "btree.h" | #include "btree.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| @@ -16,9 +16,10 @@ | |||||
| #include "minddata/dataset/util/buddy.h" | #include "minddata/dataset/util/buddy.h" | ||||
| #include <iomanip> | #include <iomanip> | ||||
| #include <stdexcept> | #include <stdexcept> | ||||
| #include "minddata/dataset/util/memory_pool.h" | #include "minddata/dataset/util/memory_pool.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/system_pool.h" | #include "minddata/dataset/util/system_pool.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "./securec.h" | #include "./securec.h" | ||||
| inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } | inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } | ||||
| @@ -19,8 +19,8 @@ | |||||
| #include <limits> | #include <limits> | ||||
| #include <utility> | #include <utility> | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/system_pool.h" | #include "minddata/dataset/util/system_pool.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -23,11 +23,11 @@ | |||||
| #include <sstream> | #include <sstream> | ||||
| #include <nlohmann/json.hpp> | #include <nlohmann/json.hpp> | ||||
| #include "minddata/dataset/util/status.h" | |||||
| #include "minddata/dataset/core/tensor.h" | #include "minddata/dataset/core/tensor.h" | ||||
| #include "minddata/dataset/core/tensor_shape.h" | #include "minddata/dataset/core/tensor_shape.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/path.h" | #include "minddata/dataset/util/path.h" | ||||
| #include "minddata/dataset/util/status.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -30,7 +30,7 @@ | |||||
| #include "minddata/dataset/core/data_type.h" | #include "minddata/dataset/core/data_type.h" | ||||
| #include "minddata/dataset/core/tensor.h" | #include "minddata/dataset/core/tensor.h" | ||||
| #include "minddata/dataset/core/tensor_shape.h" | #include "minddata/dataset/core/tensor_shape.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/path.h" | #include "minddata/dataset/util/path.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| @@ -23,12 +23,11 @@ | |||||
| #include <utility> | #include <utility> | ||||
| #include "minddata/dataset/util/allocator.h" | #include "minddata/dataset/util/allocator.h" | ||||
| #include "minddata/dataset/util/intrp_resource.h" | #include "minddata/dataset/util/intrp_resource.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/service.h" | #include "minddata/dataset/util/service.h" | ||||
| #include "minddata/dataset/util/services.h" | #include "minddata/dataset/util/services.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| using SvcAllocator = Allocator<std::pair<const std::string, IntrpResource *>>; | using SvcAllocator = Allocator<std::pair<const std::string, IntrpResource *>>; | ||||
| @@ -19,7 +19,7 @@ | |||||
| #include <iostream> | #include <iostream> | ||||
| #include <iterator> | #include <iterator> | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -14,7 +14,7 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #include "minddata/dataset/util/lock.h" | #include "minddata/dataset/util/lock.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -0,0 +1,25 @@ | |||||
| /** | |||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| */ | |||||
| #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ | |||||
| #define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ | |||||
| #ifndef ENABLE_ANDROID | |||||
| #include "utils/log_adapter.h" | |||||
| #else | |||||
| #include "mindspore/lite/src/common/log_adapter.h" | |||||
| #endif | |||||
| #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_LOG_ADAPTER_H_ | |||||
| @@ -22,8 +22,9 @@ | |||||
| #include <sstream> | #include <sstream> | ||||
| #include <utility> | #include <utility> | ||||
| #include "./securec.h" | |||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -24,9 +24,10 @@ | |||||
| #include <utility> | #include <utility> | ||||
| #include <vector> | #include <vector> | ||||
| #include "./securec.h" | |||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/allocator.h" | #include "minddata/dataset/util/allocator.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/services.h" | #include "minddata/dataset/util/services.h" | ||||
| #include "minddata/dataset/util/cond_var.h" | #include "minddata/dataset/util/cond_var.h" | ||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| @@ -28,7 +28,7 @@ | |||||
| #include "minddata/dataset/core/config_manager.h" | #include "minddata/dataset/core/config_manager.h" | ||||
| #include "minddata/dataset/core/global_context.h" | #include "minddata/dataset/core/global_context.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -22,6 +22,7 @@ | |||||
| #include <stdlib.h> | #include <stdlib.h> | ||||
| #endif | #endif | ||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include "./securec.h" | |||||
| #include "minddata/dataset/util/circular_pool.h" | #include "minddata/dataset/util/circular_pool.h" | ||||
| #include "minddata/dataset/util/random.h" | #include "minddata/dataset/util/random.h" | ||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| @@ -20,9 +20,9 @@ | |||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include <vector> | #include <vector> | ||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/path.h" | #include "minddata/dataset/util/path.h" | ||||
| #include "minddata/dataset/util/status.h" | #include "minddata/dataset/util/status.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -20,9 +20,9 @@ | |||||
| #include <stdexcept> | #include <stdexcept> | ||||
| #include <utility> | #include <utility> | ||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/path.h" | #include "minddata/dataset/util/path.h" | ||||
| #include "minddata/dataset/util/services.h" | #include "minddata/dataset/util/services.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -15,8 +15,8 @@ | |||||
| */ | */ | ||||
| #include "minddata/dataset/util/task.h" | #include "minddata/dataset/util/task.h" | ||||
| #include "utils/ms_utils.h" | #include "utils/ms_utils.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/task_manager.h" | #include "minddata/dataset/util/task_manager.h" | ||||
| #include "utils/log_adapter.h" | |||||
| #if defined(__ANDROID__) || defined(ANDROID) | #if defined(__ANDROID__) || defined(ANDROID) | ||||
| #include "minddata/dataset/util/services.h" | #include "minddata/dataset/util/services.h" | ||||
| #endif | #endif | ||||
| @@ -29,10 +29,10 @@ | |||||
| #include <thread> | #include <thread> | ||||
| #include "minddata/dataset/util/intrp_resource.h" | #include "minddata/dataset/util/intrp_resource.h" | ||||
| #include "minddata/dataset/util/list.h" | #include "minddata/dataset/util/list.h" | ||||
| #include "minddata/dataset/util/log_adapter.h" | |||||
| #include "minddata/dataset/util/memory_pool.h" | #include "minddata/dataset/util/memory_pool.h" | ||||
| #include "minddata/dataset/util/services.h" | #include "minddata/dataset/util/services.h" | ||||
| #include "minddata/dataset/util/wait_post.h" | #include "minddata/dataset/util/wait_post.h" | ||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| @@ -118,8 +118,15 @@ if (BUILD_MINDDATA STREQUAL "full") | |||||
| list(REMOVE_ITEM MINDDATA_ENGINE_OPT_PRE_SRC_FILES | list(REMOVE_ITEM MINDDATA_ENGINE_OPT_PRE_SRC_FILES | ||||
| "${MINDDATA_DIR}/engine/opt/pre/cache_transform_pass.cc" | "${MINDDATA_DIR}/engine/opt/pre/cache_transform_pass.cc" | ||||
| "${MINDDATA_DIR}/engine/opt/pre/cache_error_pass.cc" | |||||
| ) | ) | ||||
| #list(REMOVE_ITEM MINDDATA_KERNELS_IMAGE_SRC_FILES | |||||
| #"${MINDDATA_DIR}/kernels/image/affine_op.cc" | |||||
| #"${MINDDATA_DIR}/kernels/image/cutmix_batch_op.cc" | |||||
| #"${MINDDATA_DIR}/kernels/image/random_affine_op.cc" | |||||
| #) | |||||
| include_directories("${CMAKE_BINARY_DIR}/minddata/dataset/engine/cache") | include_directories("${CMAKE_BINARY_DIR}/minddata/dataset/engine/cache") | ||||
| if (BUILD_MINDDATA_EXAMPLE AND (PLATFORM_ARM32 OR PLATFORM_ARM64)) | if (BUILD_MINDDATA_EXAMPLE AND (PLATFORM_ARM32 OR PLATFORM_ARM64)) | ||||
| @@ -149,8 +156,6 @@ if (BUILD_MINDDATA STREQUAL "full") | |||||
| ${MINDDATA_EXAMPLE_SRC} | ${MINDDATA_EXAMPLE_SRC} | ||||
| ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/log_adapter.cc | ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/log_adapter.cc | ||||
| ${CORE_DIR}/utils/ms_utils.cc | ${CORE_DIR}/utils/ms_utils.cc | ||||
| ${CORE_DIR}/gvar/logging_level.cc | |||||
| ${CCSRC_DIR}/utils/system/crc32c.cc | |||||
| ) | ) | ||||
| target_link_libraries(minddata-lite | target_link_libraries(minddata-lite | ||||
| @@ -189,6 +194,10 @@ elseif (BUILD_MINDDATA STREQUAL "lite") | |||||
| ${MINDDATA_DIR}/api/execute.cc | ${MINDDATA_DIR}/api/execute.cc | ||||
| ) | ) | ||||
| list(REMOVE_ITEM MINDDATA_KERNELS_IMAGE_SRC_FILES | |||||
| "${MINDDATA_DIR}/kernels/image/bounding_box_augment_op.cc" | |||||
| "${MINDDATA_DIR}/kernels/image/cut_out_op.cc" | |||||
| ) | |||||
| add_library(minddata-lite SHARED | add_library(minddata-lite SHARED | ||||
| ${MINDDATA_CORE_SRC_FILES} | ${MINDDATA_CORE_SRC_FILES} | ||||
| ${MINDDATA_KERNELS_SRC_FILES} | ${MINDDATA_KERNELS_SRC_FILES} | ||||
| @@ -198,8 +207,6 @@ elseif (BUILD_MINDDATA STREQUAL "lite") | |||||
| ${MINDDATA_DIR}/util/memory_pool.cc | ${MINDDATA_DIR}/util/memory_pool.cc | ||||
| ${MINDDATA_DIR}/util/path.cc | ${MINDDATA_DIR}/util/path.cc | ||||
| ${MINDDATA_DIR}/api/transforms.cc | ${MINDDATA_DIR}/api/transforms.cc | ||||
| ${CORE_DIR}/utils/log_adapter.cc | |||||
| ${CORE_DIR}/gvar/logging_level.cc | |||||
| ) | ) | ||||
| target_link_libraries(minddata-lite | target_link_libraries(minddata-lite | ||||
| @@ -19,6 +19,8 @@ | |||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #include "minddata/dataset/include/tensor.h" | #include "minddata/dataset/include/tensor.h" | ||||
| #include "minddata/dataset/include/datasets.h" | #include "minddata/dataset/include/datasets.h" | ||||
| #include "minddata/dataset/include/transforms.h" | |||||
| #include "minddata/dataset/include/vision.h" | |||||
| #include "minddata/dataset/include/execute.h" | #include "minddata/dataset/include/execute.h" | ||||
| #include "minddata/dataset/util/path.h" | #include "minddata/dataset/util/path.h" | ||||