matmul and add fusion

akg

@@ -1 +1 @@
-Subproject commit f308919c39811c2c3e07fb0dcc8054a533c84cbc
+Subproject commit 46f4c28fcf2ad719f3d360a9a6dc1a7fd75d130c

mindspore/lite/src/runtime/kernel/arm/base/strided_slice.cc

@@ -40,9 +40,11 @@ void StridedSliceCPUKernel::InitFastRunParam() {
   auto in_shape = in_tensors_.front()->shape();
   auto out_shape = out_tensors_.front()->shape();
   // cal inner, outer
+  outer_ = 1;
   for (int i = 0; i < split_axis_; ++i) {
     outer_ *= in_shape[i];
   }
+  inner_ = 1;
   for (size_t i = split_axis_ + 1; i < in_shape.size(); i++) {
     inner_ *= in_shape[i];
   }

mindspore/lite/src/runtime/kernel/npu/matmul_npu.cc

@@ -15,7 +15,9 @@
  */
 
 #include "src/runtime/kernel/npu/matmul_npu.h"
+#include <memory>
 #include "src/kernel_registry.h"
+#include "src/runtime/agent/npu/npu_converter_utils.h"
 
 using mindspore::kernel::KERNEL_ARCH::kNPU;
 using mindspore::lite::KernelRegistrar;
@@ -24,6 +26,11 @@ using mindspore::schema::PrimitiveType_MatMul;
 namespace mindspore::kernel {
 int MatMulNPUKernel::IsSupport(const std::vector<lite::Tensor *> &inputs, const std::vector<lite::Tensor *> &outputs,
                                OpParameter *opParameter) {
+  if (inputs.size() == 3) {
+    if (inputs[2]->shape().size() != 1) {
+      return RET_ERROR;
+    }
+  }
   return RET_OK;
 }
 
@@ -32,19 +39,57 @@ int MatMulNPUKernel::SetNPUInputs(const std::vector<lite::Tensor *> &inputs, con
   op_ = new (std::nothrow) hiai::op::MatMul(name_);
   op_->set_input_x1(*npu_inputs[0]);
   op_->set_input_x2(*npu_inputs[1]);
+  if (npu_inputs.size() == 3) {
+    matmul_parameter_->has_bias_ = true;
+    add_op_ = new (std::nothrow) hiai::op::Add(name_ + "_add");
+    add_op_->set_input_x1(*op_);
+    auto bias_shape = inputs[2]->shape();
+    auto bias_tensor = std::make_shared<ge::Tensor>();
+    if (bias_tensor == nullptr) {
+      MS_LOG(ERROR) << "new bias_tensor failed.";
+      return RET_ERROR;
+    }
+    ge::TensorDesc bias_tensor_desc(lite::ConverterToNPUShape({1, bias_shape[0], 1, 1}), ge::FORMAT_NCHW,
+                                    lite::ConverterToNPUDataType(inputs[2]->data_type()));
+    if (outputs[0]->shape().size() == 2) {
+      bias_tensor_desc.SetShape(lite::ConverterToNPUShape({1, bias_shape[0]}));
+    }
+    bias_tensor->SetTensorDesc(bias_tensor_desc);
+    bias_tensor->SetData(reinterpret_cast<const uint8_t *>(inputs[2]->data_c()), inputs[2]->Size());
+    bias_ = new (std::nothrow) hiai::op::Const(name_ + "_bias");
+    if (bias_ == nullptr) {
+      MS_LOG(ERROR) << "new bias const failed.";
+      return RET_ERROR;
+    }
+    bias_->set_attr_value(bias_tensor);
+    add_op_->set_input_x2(*bias_);
+  }
 
   op_->set_attr_transpose_x1(matmul_parameter_->a_transpose_);
   op_->set_attr_transpose_x2(matmul_parameter_->b_transpose_);
   return RET_OK;
 }
 
-ge::Operator *mindspore::kernel::MatMulNPUKernel::GetNPUOp() { return this->op_; }
+ge::Operator *mindspore::kernel::MatMulNPUKernel::GetNPUOp() {
+  if (matmul_parameter_->has_bias_) {
+    return add_op_;
+  }
+  return op_;
+}
 
 MatMulNPUKernel::~MatMulNPUKernel() {
   if (op_ != nullptr) {
     delete op_;
     op_ = nullptr;
   }
+  if (add_op_ != nullptr) {
+    delete add_op_;
+    add_op_ = nullptr;
+  }
+  if (bias_ != nullptr) {
+    delete bias_;
+    bias_ = nullptr;
+  }
 }
 
 REG_KERNEL(kNPU, kNumberTypeFloat32, PrimitiveType_MatMul, NPUKernelCreator<MatMulNPUKernel>)

mindspore/lite/src/runtime/kernel/npu/matmul_npu.h

@@ -40,6 +40,8 @@ class MatMulNPUKernel : public NPUKernel {
 
  private:
   hiai::op::MatMul *op_ = nullptr;
+  hiai::op::Add *add_op_ = nullptr;
+  hiai::op::Const *bias_ = nullptr;
   MatMulParameter *matmul_parameter_;
 };
 }  // namespace mindspore::kernel

mindspore/lite/test/CMakeLists.txt

@@ -216,6 +216,7 @@ if(ENABLE_CONVERTER)
             ${LITE_DIR}/tools/optimizer/fusion/tflite_lstm_cell_fusion.cc
             ${LITE_DIR}/tools/optimizer/fusion/tf_lstm_cell_fusion.cc
             ${LITE_DIR}/tools/optimizer/fusion/bidirection_tf_gru_cell_fusion.cc
+            ${LITE_DIR}/tools/optimizer/fusion/matmul_add_fusion.cc
             ${LITE_DIR}/tools/optimizer/graph/weight_format_transform_pass.cc
             ${LITE_DIR}/tools/optimizer/graph/weight_format_hardcode_pass.cc
             ${LITE_DIR}/tools/optimizer/graph/conv1d_weight_expanding_pass.cc

mindspore/lite/test/models_gpu_fp32.cfg

@@ -23,4 +23,4 @@ landmark
 PoseNet_dla_17_x512
 age_new
 plat_isface
-efficientnet.mindir
+#efficientnet.mindir

mindspore/lite/tools/converter/CMakeLists.txt

@@ -49,6 +49,7 @@ file(GLOB_RECURSE CONVERTER_SRC RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
         ../optimizer/fusion/tflite_lstm_cell_fusion.cc
         ../optimizer/fusion/tf_lstm_cell_fusion.cc
         ../optimizer/fusion/bidirection_tf_gru_cell_fusion.cc
+        ../optimizer/fusion/matmul_add_fusion.cc
         ../optimizer/graph/weight_format_transform_pass.cc
         ../optimizer/graph/weight_format_hardcode_pass.cc
         ../optimizer/graph/conv1d_weight_expanding_pass.cc

mindspore/lite/tools/converter/anf_transform.cc

@@ -32,6 +32,7 @@
 #include "tools/optimizer/fusion/tflite_lstm_cell_fusion.h"
 #include "tools/optimizer/fusion/tf_lstm_cell_fusion.h"
 #include "tools/optimizer/fusion/bidirection_tf_gru_cell_fusion.h"
+#include "tools/optimizer/fusion/matmul_add_fusion.h"
 #include "tools/optimizer/graph/mindir_adjust_pass.h"
 #include "tools/optimizer/graph/mindir_inputs_adjust_pass.h"
 #include "tools/optimizer/graph/redundant_op_remove_pass.h"
@@ -104,6 +105,9 @@ int AnfTransform::AddFusionPass(const std::shared_ptr<opt::GraphOptimizer> &opti
     fusion_pm->AddPass(remove_unused_transpose_pass);
   }
   fusion_pm->AddPass(std::make_shared<opt::ConvConvFusion>());
+  if (!config->trainModel) {
+    fusion_pm->AddPass(std::make_shared<opt::MatMulAddFusion>());
+  }
   optimizer->AddPassManager(fusion_pm);
   return RET_OK;
 }

mindspore/lite/tools/converter/parser/onnx/onnx_gemm_parser.cc

@@ -1,55 +0,0 @@
-/**
- * Copyright 2020 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "tools/converter/parser/onnx/onnx_gemm_parser.h"
-#include <vector>
-#include <memory>
-
-namespace mindspore {
-namespace lite {
-lite::PrimitiveC *OnnxGemmParser::ParseLitePrimitive(const onnx::GraphProto &onnx_graph,
-                                                     const onnx::NodeProto &onnx_node) {
-  MS_LOG(DEBUG) << "onnx IdentityParser";
-  auto node_parser = OnnxNodeParserRegistry::GetInstance()->GetNodeParser("MatMul");
-  if (node_parser == nullptr) {
-    MS_LOG(ERROR) << "parse node " << onnx_node.op_type() << " failed.";
-    return nullptr;
-  }
-  auto *matmul_primitive = node_parser->ParseLitePrimitive(onnx_graph, onnx_node);
-
-  node_parser = OnnxNodeParserRegistry::GetInstance()->GetNodeParser("BiasAdd");
-  if (node_parser == nullptr) {
-    MS_LOG(ERROR) << "parse node " << onnx_node.op_type() << " failed.";
-    return nullptr;
-  }
-
-  auto *bias_add_primitive = node_parser->ParseLitePrimitive(onnx_graph, onnx_node);
-  auto primitive = std::make_unique<schema::PrimitiveT>();
-  if (primitive == nullptr) {
-    MS_LOG(ERROR) << "new primitive failed";
-    return nullptr;
-  }
-
-  primitive->value.type = schema::PrimitiveType_MakeTuple;
-  auto primitve_c = PrimitiveC::Create(primitive.release());
-  primitve_c->set_attr("MatMul", std::shared_ptr<lite::PrimitiveC>(matmul_primitive));
-  primitve_c->set_attr("BiasAdd", std::shared_ptr<lite::PrimitiveC>(bias_add_primitive));
-  return primitve_c;
-}
-
-OnnxNodeRegistrar g_onnxGemmParser("Gemm", new OnnxGemmParser());
-}  // namespace lite
-}  // namespace mindspore

mindspore/lite/tools/converter/parser/onnx/onnx_gemm_parser.h

@@ -1,34 +0,0 @@
-/**
- * Copyright 2020 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_PARSER_ONNX_GEMM_PARSER_H
-#define MINDSPORE_LITE_TOOLS_CONVERTER_PARSER_ONNX_GEMM_PARSER_H
-
-#include "tools/converter/parser/onnx/onnx_node_parser.h"
-#include "tools/converter/parser/onnx/onnx_node_parser_registry.h"
-
-namespace mindspore {
-namespace lite {
-class OnnxGemmParser : public OnnxNodeParser {
- public:
-  OnnxGemmParser() : OnnxNodeParser("Gemm") {}
-  ~OnnxGemmParser() override = default;
-
-  lite::PrimitiveC *ParseLitePrimitive(const onnx::GraphProto &onnx_graph, const onnx::NodeProto &onnx_node) override;
-};
-}  // namespace lite
-}  // namespace mindspore
-#endif  // MINDSPORE_LITE_TOOLS_CONVERTER_PARSER_ONNX_GEMM_PARSER_H

mindspore/lite/tools/converter/parser/onnx/onnx_matmul_parser.cc

@@ -58,5 +58,6 @@ lite::PrimitiveC *OnnxMatmulParser::ParseLitePrimitive(const onnx::GraphProto &o
 }
 
 OnnxNodeRegistrar g_onnxMatmulParser("MatMul", new OnnxMatmulParser());
+OnnxNodeRegistrar g_onnxGemmParser("Gemm", new OnnxMatmulParser());
 }  // namespace lite
 }  // namespace mindspore

mindspore/lite/tools/converter/parser/onnx/onnx_model_parser.cc

@@ -36,7 +36,6 @@ static const std::unordered_map<int, mindspore::TypeId> TYPE_MAP = {
   {onnx::TensorProto_DataType_FLOAT, mindspore::kNumberTypeFloat32},
   {onnx::TensorProto_DataType_BOOL, mindspore::kNumberTypeBool}};
 
-std::set<std::string> SPECIAL_NODE = {"Gemm"};
 FuncGraphPtr OnnxModelParser::Parse(const std::string &model_file, const std::string &weight_file,
                                     const QuantType &quant_type) {
   NoSupportOp::GetInstance()->SetFmkType("ONNX");
@@ -205,11 +204,6 @@ STATUS OnnxModelParser::ConvertNodes(const onnx::GraphProto &onnx_graph, const F
       MS_LOG(ERROR) << "convert " << onnx_node.op_type() << " quant param failed.";
       continue;
     }
-    if (IsSpecialOnnxNode(onnx_node)) {
-      auto status_node = ConvertSpecialOnnxNode(onnx_node, anf_graph, anf_nodes_map, primitive_c);
-      status = status == RET_OK ? status_node : status;
-      continue;
-    }
     // build CNode
     status = BuildCNode(onnx_node, anf_graph, anf_nodes_map, graph_inputs, primitive_c, root_node_name);
     if (status != RET_OK) {
@@ -1006,115 +1000,6 @@ STATUS OnnxModelParser::BuildCondGraph(const FuncGraphPtr &cond_graph, const Anf
   return status;
 }
 
-STATUS OnnxModelParser::ConvertSpecialOnnxNode(const onnx::NodeProto &onnx_node, const FuncGraphPtr &anf_graph,
-                                               std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map,
-                                               lite::PrimitiveC *primitive_c) {
-  if (primitive_c == nullptr || anf_graph == nullptr) {
-    MS_LOG(ERROR) << "imitive_c is nullptr.";
-    return RET_NULL_PTR;
-  }
-  STATUS status = RET_OK;
-  if (onnx_node.op_type() == "Gemm") {
-    status = ConvertOnnxGemmNode(onnx_node, anf_graph, anf_nodes_map, primitive_c);
-  } else {
-    MS_LOG(ERROR) << "the node is not special node.";
-    status = RET_ERROR;
-  }
-  delete primitive_c;
-  return status;
-}
-
-STATUS OnnxModelParser::ConvertOnnxGemmNode(const onnx::NodeProto &onnx_node, const FuncGraphPtr &anf_graph,
-                                            std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map,
-                                            lite::PrimitiveC *primitive_c) {
-  if (primitive_c == nullptr || anf_graph == nullptr) {
-    MS_LOG(ERROR) << "parameter has nullptr.";
-    return RET_NULL_PTR;
-  }
-  if (onnx_node.op_type() != "Gemm") {
-    MS_LOG(ERROR) << "this op is not gemm, it is " << onnx_node.op_type();
-    return RET_ERROR;
-  }
-  if (primitive_c == nullptr) {
-    MS_LOG(ERROR) << "primitive_c is nullptr.";
-    return RET_NULL_PTR;
-  }
-  auto status = BuildCNodeForGemm(onnx_node, anf_graph, anf_nodes_map, primitive_c, "MatMul");
-  if (status != RET_OK) {
-    MS_LOG(ERROR) << "convert gemm node failed.";
-    return status;
-  }
-  status = BuildCNodeForGemm(onnx_node, anf_graph, anf_nodes_map, primitive_c, "BiasAdd");
-  if (status != RET_OK) {
-    MS_LOG(ERROR) << "convert gemm node failed.";
-    return status;
-  }
-  return RET_OK;
-}
-
-STATUS OnnxModelParser::BuildCNodeForGemm(const onnx::NodeProto &onnx_node, const FuncGraphPtr &anf_graph,
-                                          std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map,
-                                          lite::PrimitiveC *primitive_c, const std::string &name) {
-  if (primitive_c == nullptr || anf_graph == nullptr) {
-    MS_LOG(ERROR) << "parameter has nullptr.";
-    return RET_NULL_PTR;
-  }
-  auto value = primitive_c->GetAttr(name);
-  primitive_c->EraseAttr(name);
-  if (value == nullptr) {
-    MS_LOG(ERROR) << "op parse failed.";
-    return RET_NULL_PTR;
-  }
-  auto prim_ptr = value->cast<std::shared_ptr<lite::PrimitiveC>>();
-  if (prim_ptr == nullptr) {
-    MS_LOG(ERROR) << "p parse failed.";
-    return RET_NULL_PTR;
-  }
-  auto type_ptr = TypeIdToType(kTypeUnknown);
-  std::vector<int64_t> shape_vector;
-  std::vector<AnfNodePtr> op_inputs;
-  if (name == "MatMul") {
-    for (int i = 0; i < 2; ++i) {
-      if (anf_nodes_map->find(onnx_node.input(i)) == anf_nodes_map->end()) {
-        MS_LOG(ERROR) << "op " << onnx_node.op_type() << " inputs get failed.";
-        return RET_ERROR;
-      } else {
-        op_inputs.push_back(anf_nodes_map->at(onnx_node.input(i)));
-        prim_ptr->AddInputQuantParam(primitive_c->input_quant_params().at(i));
-      }
-    }
-    prim_ptr->AddOutputQuantParam(std::vector<schema::QuantParamT>(1));
-    auto new_cnode = anf_graph->NewCNode(prim_ptr, op_inputs);
-    if (new_cnode == nullptr) {
-      MS_LOG(ERROR) << "new cnode error";
-      return RET_ERROR;
-    }
-    new_cnode->set_fullname_with_scope("Gemm_MatMul_" + onnx_node.output(0));
-    new_cnode->set_abstract(std::make_shared<abstract::AbstractTensor>(type_ptr, shape_vector));
-    anf_nodes_map->emplace("Gemm_MatMul_" + onnx_node.output(0), new_cnode);
-  } else {
-    if (anf_nodes_map->find("Gemm_MatMul_" + onnx_node.output(0)) == anf_nodes_map->end() ||
-        anf_nodes_map->find(onnx_node.input(2)) == anf_nodes_map->end()) {
-      MS_LOG(ERROR) << "op " << onnx_node.op_type() << " inputs get failed.";
-      return RET_ERROR;
-    }
-    op_inputs.push_back(anf_nodes_map->at("Gemm_MatMul_" + onnx_node.output(0)));
-    op_inputs.push_back(anf_nodes_map->at(onnx_node.input(2)));
-    prim_ptr->AddInputQuantParam(std::vector<schema::QuantParamT>(1));
-    prim_ptr->AddInputQuantParam(primitive_c->input_quant_params().at(2));
-    prim_ptr->AddOutputQuantParam(primitive_c->output_quant_params().front());
-    auto new_cnode = anf_graph->NewCNode(prim_ptr, op_inputs);
-    if (new_cnode == nullptr) {
-      MS_LOG(ERROR) << "new cnode error";
-      return RET_ERROR;
-    }
-    new_cnode->set_fullname_with_scope("Gemm_BiasAdd_" + onnx_node.output(0));
-    new_cnode->set_abstract(std::make_shared<abstract::AbstractTensor>(type_ptr, shape_vector));
-    anf_nodes_map->emplace(onnx_node.output(0), new_cnode);
-  }
-  return RET_OK;
-}
-
 STATUS OnnxModelParser::BuildParameterNodeForQuantParam(const void *data, const std::string &name, TypeId type) {
   if (data == nullptr) {
     MS_LOG(ERROR) << "value is nullptr.";
@@ -1262,10 +1147,6 @@ STATUS OnnxModelParser::CopyOnnxTensorData(const onnx::TensorProto &onnx_const_t
   return RET_OK;
 }
 
-bool OnnxModelParser::IsSpecialOnnxNode(const onnx::NodeProto &onnx_node) {
-  return SPECIAL_NODE.find(onnx_node.op_type()) != SPECIAL_NODE.end();
-}
-
 TypeId OnnxModelParser::GetDataTypeFromOnnx(onnx::TensorProto_DataType onnx_type) {
   auto iter = TYPE_MAP.find(onnx_type);
   if (iter == TYPE_MAP.end()) {

mindspore/lite/tools/converter/parser/onnx/onnx_model_parser.h

@@ -74,14 +74,6 @@ class OnnxModelParser : public ModelParser {
                     lite::PrimitiveC *primitive_c, std::string loop_name);
   STATUS BuildOpOutputs(const onnx::NodeProto &onnx_node, const FuncGraphPtr &func_graph_ptr,
                         std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map, const CNodePtr &cnode);
-  STATUS ConvertSpecialOnnxNode(const onnx::NodeProto &onnx_node, const FuncGraphPtr &func_graph_ptr,
-                                std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map,
-                                lite::PrimitiveC *primitive_c);
-  STATUS ConvertOnnxGemmNode(const onnx::NodeProto &onnx_node, const FuncGraphPtr &func_graph_ptr,
-                             std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map, lite::PrimitiveC *primitive_c);
-  STATUS BuildCNodeForGemm(const onnx::NodeProto &onnx_node, const FuncGraphPtr &func_graph_ptr,
-                           std::unordered_map<std::string, AnfNodePtr> *anf_nodes_map, lite::PrimitiveC *primitive_c,
-                           const std::string &name);
   STATUS ConvertOpQuantParams(const onnx::NodeProto &onnx_node, lite::PrimitiveC *primitive_c);
   STATUS ParseQuantParam(const onnx::NodeProto &onnx_node);
   STATUS SetTensorQuantParam(const std::string &tensor_name, std::vector<QuantParamT> *quant_params);

mindspore/lite/tools/converter/parser/tf/tf_batch_matmul_parser.cc

@@ -68,5 +68,6 @@ STATUS TFBatchMatMulParser::Parse(const tensorflow::NodeDef &tf_op,
   return RET_OK;
 }
 TFNodeRegistrar g_tfBatchMatMulParser("BatchMatMul", new TFBatchMatMulParser());
+TFNodeRegistrar g_tfBatchMatMulV2Parser("BatchMatMulV2", new TFBatchMatMulParser());
 }  // namespace lite
 }  // namespace mindspore

mindspore/lite/tools/optimizer/fusion/matmul_add_fusion.cc

@@ -0,0 +1,80 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "tools/optimizer/fusion/matmul_add_fusion.h"
+#include "tools/optimizer/common/gllo_utils.h"
+
+namespace mindspore {
+namespace opt {
+namespace {
+constexpr size_t AddInputSize = 3;
+constexpr size_t MatMulInputSize = 3;
+bool CheckAndGetMatMulIndex(const CNodePtr &cnode, size_t *index) {
+  MS_ASSERT(cnode != nullptr);
+  MS_ASSERT(index != nullptr);
+  if (cnode->size() != AddInputSize) {
+    return false;
+  }
+  size_t matmul_index = 0;
+  for (size_t i = 1; i < cnode->size(); ++i) {
+    if (GetCNodeType(cnode->input(i)) == schema::PrimitiveType_MatMul) {
+      auto matmul_cnode = cnode->input(i)->cast<CNodePtr>();
+      if (matmul_cnode->size() > MatMulInputSize) {
+        continue;
+      }
+      matmul_index = i;
+      break;
+    }
+  }
+  if (matmul_index == 0) {
+    return false;
+  }
+  *index = matmul_index;
+  return true;
+}
+}  // namespace
+
+bool MatMulAddFusion::Run(const FuncGraphPtr &func_graph) {
+  MS_ASSERT(func_graph != nulltr);
+  auto node_list = TopoSort(func_graph->get_return());
+  for (auto &node : node_list) {
+    if (!utils::isa<CNode>(node)) {
+      continue;
+    }
+    auto cnode = node->cast<CNodePtr>();
+    auto type = GetCNodeType(cnode);
+    if (type != schema::PrimitiveType_Add && type != schema::PrimitiveType_BiasAdd) {
+      continue;
+    }
+    size_t index = 0;
+    if (!CheckAndGetMatMulIndex(cnode, &index)) {
+      continue;
+    }
+    auto matmul_cnode = cnode->input(index)->cast<CNodePtr>();
+    auto bias_node = cnode->input(AddInputSize - index);
+    if (!utils::isa<Parameter>(bias_node) || !bias_node->cast<ParameterPtr>()->default_param()) {
+      continue;
+    }
+    matmul_cnode->add_input(bias_node);
+    auto manager = func_graph->manager();
+    MS_ASSERT(manager != nullptr);
+    matmul_cnode->set_fullname_with_scope(node->fullname_with_scope());
+    manager->Replace(node, matmul_cnode);
+  }
+  return false;
+}
+}  // namespace opt
+}  // namespace mindspore

mindspore/lite/tools/optimizer/fusion/matmul_add_fusion.h

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_PASS_FUSION_MATMUL_ADD_FUSION_H_
+#define MINDSPORE_LITE_SRC_PASS_FUSION_MATMUL_ADD_FUSION_H_
+
+#include "backend/optimizer/common/optimizer.h"
+#include "tools/converter/converter_context.h"
+#include "backend/optimizer/common/pass.h"
+
+namespace mindspore {
+namespace opt {
+class MatMulAddFusion : public Pass {
+ public:
+  MatMulAddFusion() : Pass("matmul_add_fusion") {}
+  ~MatMulAddFusion() override = default;
+  bool Run(const FuncGraphPtr &func_graph) override;
+};
+}  // namespace opt
+}  // namespace mindspore
+#endif  // MINDSPORE_LITE_SRC_PASS_FUSION_MATMUL_ADD_FUSION_H_