From b1207b2ba0cf7a120f97a0dbe01f8830bce39133 Mon Sep 17 00:00:00 2001 From: yangruoqi713 Date: Sat, 20 Feb 2021 16:18:47 +0800 Subject: [PATCH] [MSLITE][Develop] fix bug of npu kernel fusion --- .../agent/npu/optimizer/npu_fusion_pass.cc | 32 ++++++++++++------- .../optimizer/npu_insert_transform_pass.cc | 8 +++-- .../agent/npu/optimizer/npu_pass_utils.cc | 8 +++-- .../agent/npu/optimizer/npu_transform_pass.cc | 10 ++++-- .../lite/src/runtime/kernel/npu/resize_npu.cc | 15 ++++++--- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_fusion_pass.cc b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_fusion_pass.cc index 1bc0b067d4..aba0a415a3 100644 --- a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_fusion_pass.cc +++ b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_fusion_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -197,7 +197,11 @@ int NPUFusionPass::ConcatFusion(kernel::LiteKernel *kernel) { } int NPUFusionPass::FormatFusion(kernel::LiteKernel *kernel) { - auto pre_kernel = kernel->in_kernels()[0]; + auto is_input_kernel = kernel->in_kernels().empty(); + kernel::LiteKernel *pre_kernel = nullptr; + if (!is_input_kernel) { + pre_kernel = kernel->in_kernels()[0]; + } auto in_tensor = kernel->in_tensors()[0]; std::vector pre_insert_kernels; for (const auto &trans_kernel : kernel->out_kernels()) { @@ -225,7 +229,11 @@ int NPUFusionPass::FormatFusion(kernel::LiteKernel *kernel) { auto post_in_kernels = post_kernel->in_kernels(); for (size_t i = 0; i < post_in_kernels.size(); i++) { if (post_in_kernels[i] == trans_kernel) { - post_in_kernels[i] = pre_kernel; + if (is_input_kernel) { + post_in_kernels.erase(post_in_kernels.begin() + i); + } else { + post_in_kernels[i] = pre_kernel; + } break; } } @@ -234,16 +242,18 @@ int NPUFusionPass::FormatFusion(kernel::LiteKernel *kernel) { } RemoveAndFreeKernel(trans_kernel); } - auto pre_out_kernels = pre_kernel->out_kernels(); - size_t index = 0; - for (; index < pre_out_kernels.size(); index++) { - if (pre_out_kernels[index] == kernel) { - pre_out_kernels.erase(pre_out_kernels.begin() + index); - break; + if (!is_input_kernel) { + auto pre_out_kernels = pre_kernel->out_kernels(); + size_t index = 0; + for (; index < pre_out_kernels.size(); index++) { + if (pre_out_kernels[index] == kernel) { + pre_out_kernels.erase(pre_out_kernels.begin() + index); + break; + } } + pre_out_kernels.insert(pre_out_kernels.begin() + index, pre_insert_kernels.begin(), pre_insert_kernels.end()); + pre_kernel->set_out_kernels(pre_out_kernels); } - pre_out_kernels.insert(pre_out_kernels.begin() + index, pre_insert_kernels.begin(), pre_insert_kernels.end()); - pre_kernel->set_out_kernels(pre_out_kernels); RemoveAndFreeKernel(kernel); return RET_OK; } diff --git a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_insert_transform_pass.cc b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_insert_transform_pass.cc index 8eb75aebac..83785055c8 100644 --- a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_insert_transform_pass.cc +++ b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_insert_transform_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -116,28 +116,30 @@ int NPUInsertTransformPass::InsertNode(kernel::LiteKernel *kernel, kernel::LiteK std::vector nhwc_shape = in_tensor->shape(); std::vector nchw_shape = {nhwc_shape[0], nhwc_shape[3], nhwc_shape[1], nhwc_shape[2]}; + auto nh2nc_name = kernel_name + "_nh2nc_" + std::to_string(total++); auto nh2nc_tensor = new (std::nothrow) Tensor(in_tensor->data_type(), nchw_shape, schema::Format_NHWC, Tensor::VAR); if (nh2nc_tensor == nullptr) { MS_LOG(ERROR) << "New nchw tensor failed when inserting nchw2nhwc kernel."; return RET_ERROR; } + nh2nc_tensor->set_tensor_name(nh2nc_name + "/output0"); std::vector nh2nc_tensors = {nh2nc_tensor}; all_tensors_->push_back(nh2nc_tensors[0]); + auto nc2nh_name = kernel_name + "_nc2nh_" + std::to_string(total++); auto nc2nh_tensor = new (std::nothrow) Tensor(in_tensor->data_type(), nhwc_shape, schema::Format_NCHW, Tensor::VAR); if (nc2nh_tensor == nullptr) { MS_LOG(ERROR) << "New nhwc tensor failed when inserting nhwc2nchw kernel."; return RET_ERROR; } + nc2nh_tensor->set_tensor_name(nc2nh_name + "/output0"); std::vector nc2nh_tensors = {nc2nh_tensor}; all_tensors_->push_back(nc2nh_tensors[0]); - auto nh2nc_name = kernel_name + "_nh2nc_" + std::to_string(total++); auto *nh2nc_kernel = NPUPassUtils::CreateNhwc2NchwKernel({in_tensor}, nh2nc_tensors, context_, nh2nc_name); trans_kernels->push_back(nh2nc_kernel); insert_primitive_.push_back(nh2nc_kernel->GetPrimitive()); - auto nc2nh_name = kernel_name + "_nc2nh_" + std::to_string(total++); auto *nc2nh_kernel = NPUPassUtils::CreateNchw2NhwcKernel(nh2nc_tensors, nc2nh_tensors, context_, nc2nh_name); trans_kernels->push_back(nc2nh_kernel); insert_primitive_.push_back(nc2nh_kernel->GetPrimitive()); diff --git a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_pass_utils.cc b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_pass_utils.cc index b9299e5930..139df3216d 100644 --- a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_pass_utils.cc +++ b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_pass_utils.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -191,7 +191,11 @@ void NPUPassUtils::UpdateNC2NHTransNodePostKernel(kernel::LiteKernel *kernel, ke // For post_kernel after trans, kernel in in_kernels should be replaced with trans_kernel. auto post_in_kernels = post_kernel->in_kernels(); - std::replace(post_in_kernels.begin(), post_in_kernels.end(), kernel, trans_kernel); + if (kernel == nullptr) { + post_in_kernels.push_back(trans_kernel); + } else { + std::replace(post_in_kernels.begin(), post_in_kernels.end(), kernel, trans_kernel); + } post_kernel->set_in_kernels(post_in_kernels); } diff --git a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_transform_pass.cc b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_transform_pass.cc index 300439c948..fbabd1e25c 100644 --- a/mindspore/lite/src/runtime/agent/npu/optimizer/npu_transform_pass.cc +++ b/mindspore/lite/src/runtime/agent/npu/optimizer/npu_transform_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,11 +39,12 @@ int NPUTransformPass::InsertPreNodes(kernel::LiteKernel *kernel, std::vectorname() + "_pre_trans" + "_Nhwc2Nchw_" + std::to_string(total++); + tensor->set_tensor_name(name + "/output0"); std::vector pre_trans_out_tensors = {tensor}; all_tensors_->push_back(pre_trans_out_tensors[0]); // Create pre transform kernel: Nhwc2Nchw - auto name = kernel->name() + "_pre_trans" + "_Nhwc2Nchw_" + std::to_string(total++); auto *trans_kernel = NPUPassUtils::CreateNhwc2NchwKernel({kernel->in_tensors()[0]}, pre_trans_out_tensors, context_, name); @@ -124,6 +125,11 @@ int NPUTransformPass::Run() { i++; continue; } + if (kernel->Type() == schema::PrimitiveType_Resize && + kernel->in_tensors()[0]->Height() > kernel->out_tensors()[0]->Height()) { + i++; + continue; + } // insert pre_kernels before kernel in vector // modify loop index add (pre_kernels.size() + 1) to the post_kernels insert location std::vector pre_kernels; diff --git a/mindspore/lite/src/runtime/kernel/npu/resize_npu.cc b/mindspore/lite/src/runtime/kernel/npu/resize_npu.cc index 29bcd0ac10..e17f054d48 100644 --- a/mindspore/lite/src/runtime/kernel/npu/resize_npu.cc +++ b/mindspore/lite/src/runtime/kernel/npu/resize_npu.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,11 +26,15 @@ using mindspore::schema::PrimitiveType_Resize; namespace mindspore::kernel { int ResizeNPUKernel::IsSupport(const std::vector &inputs, const std::vector &outputs, OpParameter *opParameter) { - if (resize_parameter_->method_ != schema::ResizeMethod_LINEAR || - resize_parameter_->method_ == schema::ResizeMethod_NEAREST) { + if (resize_parameter_->method_ != schema::ResizeMethod_LINEAR && + resize_parameter_->method_ != schema::ResizeMethod_NEAREST) { MS_LOG(WARNING) << "Unsupported resize method type:" << resize_parameter_->method_; return RET_ERROR; } + if (inputs[0]->Height() > outputs[0]->Height() || inputs[0]->Width() > outputs[0]->Width()) { + MS_LOG(WARNING) << "Npu resize does not support reduction."; + return RET_ERROR; + } return RET_OK; } @@ -55,7 +59,7 @@ int ResizeNPUKernel::SetNPUInputs(const std::vector &inputs, con op->set_input_size(*out_size); op->set_attr_half_pixel_centers(resize_parameter_->preserve_aspect_ratio_); op_ = op; - } else { + } else if (resize_parameter_->method_ == schema::ResizeMethod_NEAREST) { auto op = new (std::nothrow) hiai::op::ResizeNearestNeighborV2(name_); if (op == nullptr) { MS_LOG(ERROR) << " op is nullptr."; @@ -66,6 +70,9 @@ int ResizeNPUKernel::SetNPUInputs(const std::vector &inputs, con op->set_input_x(*npu_inputs[0]); op->set_input_size(*out_size); op_ = op; + } else { + MS_LOG(WARNING) << "Unsupported resize method type:" << resize_parameter_->method_; + return RET_ERROR; } return RET_OK; }