diff --git a/mindspore/ccsrc/backend/optimizer/graph_kernel/depend_formater.cc b/mindspore/ccsrc/backend/optimizer/graph_kernel/depend_formater.cc
index 10a778ae5a..9e0211042d 100644
--- a/mindspore/ccsrc/backend/optimizer/graph_kernel/depend_formater.cc
+++ b/mindspore/ccsrc/backend/optimizer/graph_kernel/depend_formater.cc
@@ -135,7 +135,14 @@ bool DependFormater::Run(const FuncGraphPtr &func_graph) {
     }
 
     old_depends.push_back(node);
-    free_nodes.push_back(node->cast<CNodePtr>()->input(kDependAttachNodeIndex));
+    auto cnode = node->cast<CNodePtr>();
+    for (size_t id = kDependAttachNodeIndex; id < cnode->inputs().size(); ++id) {
+      auto attach_node = cnode->input(id);
+      if (!IsPrimitiveCNode(attach_node, prim::kPrimDepend)) {
+        continue;
+      }
+      free_nodes.push_back(attach_node);
+    }
   }
 
   if (old_depends.empty()) {
diff --git a/mindspore/ccsrc/backend/optimizer/graph_kernel/parallel_fusion.cc b/mindspore/ccsrc/backend/optimizer/graph_kernel/parallel_fusion.cc
index 73e79c3cdd..a1c01203e0 100644
--- a/mindspore/ccsrc/backend/optimizer/graph_kernel/parallel_fusion.cc
+++ b/mindspore/ccsrc/backend/optimizer/graph_kernel/parallel_fusion.cc
@@ -96,6 +96,7 @@ void ProcessThroughPassCNode(std::function<bool(const AnfNodePtr &)> pass_fn,
 }
 
 void ProcessDependCNode(OrderedMap<AnfNodePtr, NodeRelation> *node_rels) {
+  OrderedSet<AnfNodePtr> to_be_through_pass;
   for (auto &[node, node_rel] : (*node_rels)) {
     if (!IsPrimitiveCNode(node, prim::kPrimDepend) ||
         HasAbstractMonad(node->cast<CNodePtr>()->input(kDependAttachNodeIndex))) {
@@ -113,10 +114,12 @@ void ProcessDependCNode(OrderedMap<AnfNodePtr, NodeRelation> *node_rels) {
         cnode_pres.erase(attach_node);
       }
     }
+    to_be_through_pass.insert(node);
   }
 
   // Eliminate depend node of node relations.
-  ProcessThroughPassCNode([](const AnfNodePtr &node) { return IsOneOf(node, {prim::kPrimDepend}); }, node_rels);
+  ProcessThroughPassCNode([&to_be_through_pass](const AnfNodePtr &node) { return to_be_through_pass.count(node) > 0; },
+                          node_rels);
 }
 
 void ProcessTailMakeTupleCNode(OrderedMap<AnfNodePtr, NodeRelation> *node_rels) {