!12460 [MS_LITE] fix fp16 conv data type

From: @YeFeng_24
Reviewed-by: @hangangqiang,@zhanghaibo5
Signed-off-by: @hangangqiang

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.cc

@@ -93,8 +93,7 @@ int Convolution1x1FP16CPUKernel::InitWeightBias() {
       MS_LOG(ERROR) << "Conv1x1 Malloc bias_ptr_ error!";
       return RET_ERROR;
     }
-    auto bias_tensor = in_tensors_.at(kBiasIndex);
-    if (bias_tensor->data_type() == kNumberTypeFloat16) {
+    if (origin_bias_data_type_ == kNumberTypeFloat16) {
       memcpy(bias_data_, origin_bias_, output_channel * sizeof(float16_t));
     } else {
       Float32ToFloat16(reinterpret_cast<float *>(origin_bias_), reinterpret_cast<float16_t *>(bias_data_),
@@ -112,7 +111,7 @@ int Convolution1x1FP16CPUKernel::InitWeightBias() {
   }
   memset(reinterpret_cast<char *>(weight_ptr_) + down_size, 0, size - down_size);
   ColMajor2Row8MajorFp16(origin_weight_, weight_ptr_, input_channel, output_channel,
-                         weight_tensor->data_type() == kNumberTypeFloat16);
+                         origin_weight_data_type_ == kNumberTypeFloat16);
   return RET_OK;
 }
 

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.h

@@ -30,10 +30,13 @@ class Convolution1x1FP16CPUKernel : public ConvolutionBaseFP16CPUKernel {
  public:
   Convolution1x1FP16CPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
                               const std::vector<lite::Tensor *> &outputs, const InnerContext *ctx,
-                              const mindspore::lite::PrimitiveC *primitive, void *origin_weight, void *origin_bias)
+                              const mindspore::lite::PrimitiveC *primitive, void *origin_weight, void *origin_bias,
+                              TypeId origin_weight_data_type, TypeId origin_bias_data_type)
       : ConvolutionBaseFP16CPUKernel(parameter, inputs, outputs, ctx, primitive),
         origin_weight_(origin_weight),
-        origin_bias_(origin_bias) {}
+        origin_bias_(origin_bias),
+        origin_weight_data_type_(origin_weight_data_type),
+        origin_bias_data_type_(origin_bias_data_type) {}
   ~Convolution1x1FP16CPUKernel() override;
 
   int Init() override;
@@ -62,6 +65,8 @@ class Convolution1x1FP16CPUKernel : public ConvolutionBaseFP16CPUKernel {
   float16_t *pack_input_ = nullptr;
   float16_t *output_ptr_ = nullptr;
   MatMulParameter *matmul_param_ = nullptr;
+  TypeId origin_weight_data_type_;
+  TypeId origin_bias_data_type_;
 };
 }  // namespace mindspore::kernel
 

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_delegate_fp16.cc

@@ -67,15 +67,17 @@ int ConvolutionDelegateFP16CPUKernel::Init() {
     if (in_tensors_.size() == 3) {
       origin_bias_ = CopyData(in_tensors_.at(kBiasIndex));
       need_free_ = need_free_ | BIAS_NEED_FREE;
+      origin_bias_data_type_ = in_tensors_.at(kBiasIndex)->data_type();
     }
-    origin_weight_data_type_ = in_tensors_[1]->data_type();
+    origin_weight_data_type_ = in_tensors_.at(kWeightIndex)->data_type();
     return RET_OK;
   }
   origin_weight_ = in_tensors_.at(kWeightIndex)->data_c();
   if (in_tensors_.size() == 3) {
     origin_bias_ = in_tensors_.at(kBiasIndex)->data_c();
+    origin_bias_data_type_ = in_tensors_.at(kBiasIndex)->data_type();
   }
-  origin_weight_data_type_ = in_tensors_[1]->data_type();
+  origin_weight_data_type_ = in_tensors_.at(kWeightIndex)->data_type();
   return ReSize();
 }
 
@@ -84,8 +86,9 @@ int ConvolutionDelegateFP16CPUKernel::ReSize() {
   SetInputOutputShapeInfo(reinterpret_cast<ConvParameter *>(op_parameter_), in_tensors_.front(), out_tensors_.front(),
                           context_);
   if (fp16_conv_kernel_ == nullptr) {
-    fp16_conv_kernel_ = CpuConvFp16KernelSelect(in_tensors_, out_tensors_, op_parameter_, context_, primitive_,
-                                                origin_weight_, origin_bias_, origin_weight_data_type_);
+    fp16_conv_kernel_ =
+      CpuConvFp16KernelSelect(in_tensors_, out_tensors_, op_parameter_, context_, primitive_, origin_weight_,
+                              origin_bias_, origin_weight_data_type_, origin_bias_data_type_);
     if (fp16_conv_kernel_ == nullptr) {
       MS_LOG(ERROR) << "Selecting execute kernel failed for conv_kernel, got a nullptr.";
       return RET_ERROR;
@@ -109,7 +112,8 @@ ConvParameter *CreateNewConvParameterFp16(ConvParameter *parameter) {
 kernel::LiteKernel *CpuConvFp16KernelSelect(const std::vector<lite::Tensor *> &inputs,
                                             const std::vector<lite::Tensor *> &outputs, OpParameter *op_parameter,
                                             const lite::InnerContext *ctx, const mindspore::lite::PrimitiveC *primitive,
-                                            void *origin_weight, void *origin_bias, TypeId origin_weight_data_type) {
+                                            void *origin_weight, void *origin_bias, TypeId origin_weight_data_type,
+                                            TypeId origin_bias_data_type) {
   auto conv_param = reinterpret_cast<ConvParameter *>(op_parameter);
   bool use_winograd = false;
   int out_unit;
@@ -117,13 +121,15 @@ kernel::LiteKernel *CpuConvFp16KernelSelect(const std::vector<lite::Tensor *> &i
   kernel::LiteKernel *kernel = nullptr;
   if (conv_param->kernel_h_ == 1 && conv_param->kernel_w_ == 1) {
     kernel = new (std::nothrow)
-      kernel::Convolution1x1FP16CPUKernel(op_parameter, inputs, outputs, ctx, primitive, origin_weight, origin_bias);
+      kernel::Convolution1x1FP16CPUKernel(op_parameter, inputs, outputs, ctx, primitive, origin_weight, origin_bias,
+                                          origin_weight_data_type, origin_bias_data_type);
   } else if (use_winograd) {
-    kernel = new (std::nothrow) kernel::ConvolutionWinogradFP16CPUKernel(op_parameter, inputs, outputs, ctx, primitive,
-                                                                         out_unit, origin_weight, origin_bias);
+    kernel = new (std::nothrow) kernel::ConvolutionWinogradFP16CPUKernel(
+      op_parameter, inputs, outputs, ctx, primitive, out_unit, origin_weight, origin_bias, origin_bias_data_type);
   } else {
-    kernel = new (std::nothrow) kernel::ConvolutionFP16CPUKernel(op_parameter, inputs, outputs, ctx, primitive,
-                                                                 origin_weight, origin_bias, origin_weight_data_type);
+    kernel =
+      new (std::nothrow) kernel::ConvolutionFP16CPUKernel(op_parameter, inputs, outputs, ctx, primitive, origin_weight,
+                                                          origin_bias, origin_weight_data_type, origin_bias_data_type);
   }
   // Once kernel is selected, init func will invoke InitWeightAndBias
   auto ret = kernel->Init();

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_delegate_fp16.h

@@ -55,12 +55,14 @@ class ConvolutionDelegateFP16CPUKernel : public LiteKernel {
   void *origin_bias_ = nullptr;
   kernel::LiteKernel *fp16_conv_kernel_ = nullptr;
   TypeId origin_weight_data_type_;
+  TypeId origin_bias_data_type_;
 };
 
 kernel::LiteKernel *CpuConvFp16KernelSelect(const std::vector<lite::Tensor *> &inputs,
                                             const std::vector<lite::Tensor *> &outputs, OpParameter *op_parameter,
                                             const lite::InnerContext *ctx, const mindspore::lite::PrimitiveC *primitive,
-                                            void *origin_weight, void *origin_bias, TypeId origin_weight_data_type);
+                                            void *origin_weight, void *origin_bias, TypeId origin_weight_data_type,
+                                            TypeId origin_bias_data_type);
 }  // namespace mindspore::kernel
 
 #endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DELEGATE_FP16_H_

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.cc

@@ -62,8 +62,7 @@ int ConvolutionFP16CPUKernel::InitWeightBias() {
   }
   memset(bias_data_, 0, oc8 * sizeof(float16_t));
   if (in_tensors_.size() == kInputSize2) {
-    auto bias_tensor = in_tensors_.at(kBiasIndex);
-    if (bias_tensor->data_type() == kNumberTypeFloat16) {
+    if (origin_bias_data_type_ == kNumberTypeFloat16) {
       memcpy(bias_data_, origin_bias_, out_channel * sizeof(float16_t));
     } else {
       Float32ToFloat16(reinterpret_cast<float *>(origin_bias_), reinterpret_cast<float16_t *>(bias_data_), out_channel);

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.h

@@ -28,11 +28,12 @@ class ConvolutionFP16CPUKernel : public ConvolutionBaseFP16CPUKernel {
   ConvolutionFP16CPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
                            const std::vector<lite::Tensor *> &outputs, const InnerContext *ctx,
                            const mindspore::lite::PrimitiveC *primitive, void *origin_weight, void *origin_bias,
-                           TypeId origin_weight_data_type)
+                           TypeId origin_weight_data_type, TypeId origin_bias_data_type)
       : ConvolutionBaseFP16CPUKernel(parameter, inputs, outputs, ctx, primitive),
         origin_weight_(origin_weight),
         origin_bias_(origin_bias),
-        origin_weight_data_type_(origin_weight_data_type) {}
+        origin_weight_data_type_(origin_weight_data_type),
+        origin_bias_data_type_(origin_bias_data_type) {}
   ~ConvolutionFP16CPUKernel() override {
     if (packed_weight_ != nullptr) {
       free(packed_weight_);
@@ -65,6 +66,7 @@ class ConvolutionFP16CPUKernel : public ConvolutionBaseFP16CPUKernel {
   float16_t *packed_weight_ = nullptr;
   float16_t *col_major_input_ = nullptr;
   TypeId origin_weight_data_type_;
+  TypeId origin_bias_data_type_;
 };
 }  // namespace mindspore::kernel
 

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.cc

@@ -93,8 +93,7 @@ int ConvolutionWinogradFP16CPUKernel::InitWeightBias() {
   memset(bias_data_, 0, oc_block_num * oc_block * sizeof(float16_t));
 
   if (in_tensors_.size() == kInputSize2) {
-    auto bias_tensor = in_tensors_.at(kBiasIndex);
-    if (bias_tensor->data_type() == kNumberTypeFloat16) {
+    if (origin_bias_data_type_ == kNumberTypeFloat16) {
       memcpy(bias_data_, origin_bias_, out_channel * sizeof(float16_t));
     } else {
       Float32ToFloat16(reinterpret_cast<float *>(origin_bias_), reinterpret_cast<float16_t *>(bias_data_), out_channel);

mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.h

@@ -32,11 +32,12 @@ class ConvolutionWinogradFP16CPUKernel : public ConvolutionBaseFP16CPUKernel {
   ConvolutionWinogradFP16CPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
                                    const std::vector<lite::Tensor *> &outputs, const InnerContext *ctx,
                                    const mindspore::lite::PrimitiveC *primitive, int out_unit, void *origin_weight,
-                                   void *origin_bias)
+                                   void *origin_bias, TypeId origin_bias_data_type)
       : ConvolutionBaseFP16CPUKernel(parameter, inputs, outputs, ctx, primitive),
         output_unit_(out_unit),
         origin_weight_(origin_weight),
-        origin_bias_(origin_bias) {}
+        origin_bias_(origin_bias),
+        origin_bias_data_type_(origin_bias_data_type) {}
   ~ConvolutionWinogradFP16CPUKernel() override {
     if (trans_weight_ != nullptr) {
       free(trans_weight_);
@@ -86,6 +87,7 @@ class ConvolutionWinogradFP16CPUKernel : public ConvolutionBaseFP16CPUKernel {
   TmpBufferAddressFp16 tmp_buffer_address_list_[4];
   InputTransFp16Func in_func_;
   OutputTransFp16Func out_func_;
+  TypeId origin_bias_data_type_;
 };
 }  // namespace mindspore::kernel