Merge remote-tracking branch 'upstream/development'

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "metadef"]
 	path = metadef
 	url = https://gitee.com/ascend/metadef.git
+	branch = development

build.sh

@@ -86,6 +86,8 @@ checkopts()
 }
 checkopts "$@"
 
+git submodule update --init metadef
+
 mk_dir() {
     local create_dir="$1"  # the target to make
 
@@ -152,12 +154,41 @@ generate_package()
   cd "${BASEPATH}"
 
   PARSER_LIB_PATH="lib"
+  ACL_PATH="acllib/lib64"
+  FWK_PATH="fwkacllib/lib64"
+  ATC_PATH="atc/lib64"
+
+  COMMON_LIB=("libgraph.so" "libregister.so")
+  PARSER_LIB=("lib_caffe_parser.so" "libfmk_onnx_parser.so" "libfmk_parser.so" "libparser_common.so")
+
+  rm -rf ${OUTPUT_PATH:?}/${FWK_PATH}/
+  rm -rf ${OUTPUT_PATH:?}/${ACL_PATH}/
+  rm -rf ${OUTPUT_PATH:?}/${ATC_PATH}/
+
+  mk_dir "${OUTPUT_PATH}/${FWK_PATH}"
+  mk_dir "${OUTPUT_PATH}/${ATC_PATH}"
+  mk_dir "${OUTPUT_PATH}/${ACL_PATH}"
 
   find output/ -name parser_lib.tar -exec rm {} \;
  
   cd "${OUTPUT_PATH}"
 
-  tar -cf parser_lib.tar "${PARSER_LIB_PATH}"
+  for lib in "${PARSER_LIB[@]}";
+  do
+    find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH} \;
+    find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \;
+  done
+
+  for lib in "${COMMON_LIB[@]}";
+  do
+    find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH} \;
+    find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \;
+  done
+
+  find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "libc_sec.so" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \;
+  find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "libregister.a" -exec cp -f {} ${OUTPUT_PATH}/${ACL_PATH} \;
+
+  tar -cf parser_lib.tar fwkacllib acllib atc
 }
 
 if [[ "X$ENABLE_GE_UT" = "Xoff" ]]; then

metadef

@@ -1 +1 @@
-Subproject commit 765d85777ec10fe819799cd2c1b60db49be7c749
+Subproject commit cc9de48a7779cf95cab90a23db608421a691fd12

parser/CMakeLists.txt

@@ -59,24 +59,12 @@ target_include_directories(fmk_parser PRIVATE
     ${PARSER_DIR}
     ${PARSER_DIR}/inc
     ${PARSER_DIR}/parser
-    ${PARSER_DIR}/../ge
-    ${PARSER_DIR}/../inc
-    ${PARSER_DIR}/../inc/framework
-    ${PARSER_DIR}/../inc/common/util
-    ${PARSER_DIR}/../inc/external
-    ${PARSER_DIR}/../third_party/fwkacllib/inc
     ${METADEF_DIR}/inc
     ${METADEF_DIR}/inc/graph
     ${METADEF_DIR}/inc/register
     ${METADEF_DIR}/inc/external
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/external/register
-    #### independent compile #####
-    ${METADEF_DIR}/third_party/graphengine/ge
-    ${METADEF_DIR}/third_party/graphengine/inc
-    ${METADEF_DIR}/third_party/graphengine/inc/framework
-    ${METADEF_DIR}/third_party/graphengine/inc/external
-    ${METADEF_DIR}/third_party/fwkacllib/inc
     #### temp ####
     ${PARSER_DIR}/../graphengine/inc/common/util
     ${PARSER_DIR}/../graphengine/inc/external
@@ -84,7 +72,20 @@ target_include_directories(fmk_parser PRIVATE
     ${PARSER_DIR}/../graphengine/inc
     ${PARSER_DIR}/../graphengine/ge
     ${CMAKE_BINARY_DIR}
-    ${CMAKE_BINARY_DIR}/proto/ge 
+    ${CMAKE_BINARY_DIR}/proto/ge
+    #### blue zone compile #####
+    ${PARSER_DIR}/../ge
+    ${PARSER_DIR}/../inc
+    ${PARSER_DIR}/../inc/framework
+    ${PARSER_DIR}/../inc/common/util
+    ${PARSER_DIR}/../inc/external
+    ${PARSER_DIR}/../third_party/fwkacllib/inc
+    #### blue independent compile #####
+    ${METADEF_DIR}/third_party/graphengine/ge
+    ${METADEF_DIR}/third_party/graphengine/inc
+    ${METADEF_DIR}/third_party/graphengine/inc/framework
+    ${METADEF_DIR}/third_party/graphengine/inc/external
+    ${METADEF_DIR}/third_party/fwkacllib/inc
 )
 
 target_link_libraries(fmk_parser

parser/caffe/caffe_parser.cc

@@ -20,6 +20,7 @@
 #include <iostream>
 #include <sstream>
 #include <memory>
+#include <algorithm>
 #include "parser/common/convert/pb2json.h"
 #include "common/debug/log.h"
 #include "parser/common/acl_graph_parser_util.h"
@@ -1539,6 +1540,34 @@ void CaffeModelParser::SaveOrigionLayerTops(domi::caffe::LayerParameter &layer)
   return;
 }
 
+Status CaffeModelParser::SaveDataLayerTops(const domi::caffe::LayerParameter &layer) {
+  string name = layer.name();
+  if (node_map.find(name) == node_map.end()) {
+    GELOGE(FAILED, "Node can not be found by layer name: %s", name.c_str());
+    return FAILED;
+  }
+
+  ge::NodePtr node = node_map[name];
+  GE_CHECK_NOTNULL(node);
+
+  if (node->GetType() == ge::parser::DATA) {
+    if (layer.top_size() != 1) {
+      GELOGE(FAILED, "Data layer[%s] top size must be 1, real size: %d", name.c_str(), layer.top_size());
+      return FAILED;
+    }
+
+    string top_name = layer.top(0);
+    auto data_top_names = ge::GetParserContext().data_top_names;
+    if (find(data_top_names.begin(), data_top_names.end(), top_name) != data_top_names.end()) {
+      GELOGE(FAILED, "Different data can not have same top name: %s.", top_name.c_str());
+      return FAILED;
+    }
+    ge::GetParserContext().data_top_names.push_back(top_name);
+  }
+
+  return SUCCESS;
+}
+
 Status CaffeModelParser::Parse(const char *model_path, ge::ComputeGraphPtr &graph) {
   bool has_error = false;
   GE_CHECK_NOTNULL(model_path);
@@ -1610,6 +1639,7 @@ Status CaffeModelParser::Parse(const char *model_path, ge::ComputeGraphPtr &grap
   // Map of operator name and occurrence times
   std::map<std::string, int32_t> layer_name_map;
 
+  GetParserContext().data_top_names.clear();
   // <layername,paramnames>
   std::map<std::string, std::vector<std::string>> layer_params_map;
   // same param name set <paramnames,layernames>
@@ -1656,6 +1686,10 @@ Status CaffeModelParser::Parse(const char *model_path, ge::ComputeGraphPtr &grap
 
     GE_RETURN_WITH_LOG_IF_ERROR(AddBlobsToMap(layer, inplace_blob_name_remapping),
                                 "Caffe parser add blobs to map ret fail.");
+    if (SaveDataLayerTops(layer) != SUCCESS) {
+      GELOGE(FAILED, "Caffe parse: save data layer tops failed.");
+      return FAILED;
+    }
   }
   // Find a layer with the same param name and save it to graph
   GE_RETURN_WITH_LOG_IF_ERROR(FindShareParamLayers(layer_params_map),

parser/caffe/caffe_parser.h

@@ -343,6 +343,8 @@ class CaffeModelParser : public domi::ModelParser {
 
   Status ParseOutputNodeTopInfo(const domi::caffe::NetParameter &proto_message);
 
+  Status SaveDataLayerTops(const domi::caffe::LayerParameter &layer);
+
   std::map<std::string, ge::NodePtr> node_map;
 
   // key: blob name, value: layer name and index

parser/common/CMakeLists.txt

@@ -41,24 +41,12 @@ target_include_directories(parser_common PRIVATE
     ${CMAKE_CURRENT_LIST_DIR}
     ${PARSER_DIR}
     ${PARSER_DIR}/parser
-    ${PARSER_DIR}/../ge
-    ${PARSER_DIR}/../inc
-    ${PARSER_DIR}/../inc/framework
-    ${PARSER_DIR}/../inc/common/util
-    ${PARSER_DIR}/../inc/external
-    ${PARSER_DIR}/../third_party/fwkacllib/inc
     ${METADEF_DIR}/inc
     ${METADEF_DIR}/inc/graph
     ${METADEF_DIR}/inc/register
     ${METADEF_DIR}/inc/external
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/external/register
-    #### independent compile #####
-    ${METADEF_DIR}/third_party/graphengine/ge
-    ${METADEF_DIR}/third_party/graphengine/inc
-    ${METADEF_DIR}/third_party/graphengine/inc/framework
-    ${METADEF_DIR}/third_party/graphengine/inc/external
-    ${METADEF_DIR}/third_party/fwkacllib/inc
     #### temp ####
     ${PARSER_DIR}/../graphengine/inc/common/util
     ${PARSER_DIR}/../graphengine/inc/external
@@ -67,6 +55,19 @@ target_include_directories(parser_common PRIVATE
     ${PARSER_DIR}/../graphengine/ge
     ${CMAKE_BINARY_DIR}
     ${CMAKE_BINARY_DIR}/proto/ge
+    #### blue zone compile #####
+    ${PARSER_DIR}/../ge
+    ${PARSER_DIR}/../inc
+    ${PARSER_DIR}/../inc/framework
+    ${PARSER_DIR}/../inc/common/util
+    ${PARSER_DIR}/../inc/external
+    ${PARSER_DIR}/../third_party/fwkacllib/inc
+    #### independent compile #####
+    ${METADEF_DIR}/third_party/graphengine/ge
+    ${METADEF_DIR}/third_party/graphengine/inc
+    ${METADEF_DIR}/third_party/graphengine/inc/framework
+    ${METADEF_DIR}/third_party/graphengine/inc/external
+    ${METADEF_DIR}/third_party/fwkacllib/inc
 )
 
 target_link_libraries(parser_common PRIVATE

parser/common/proto/insert_op.proto

@@ -45,6 +45,9 @@ message AippOpParams {
 	// 标识对模型的第几个输入做AIPP处理，例如模型有两个输入，需要对第2个输入做AIPP，则配置related_input_rank为1。
 	uint32 related_input_rank = 2;
 
+        // related_input_name is optional and the top name of data node which inserts aipp
+        string related_input_name = 6;
+
 	// input_edge_idx参数为可选，类型为整型，配置范围为>=0。
 	// 配置该参数的作用，在于对Data算子不同的输出做不同的AIPP处理，如果该参数没有配置，默认对related_input_rank指定的模型输入的所有输出边做AIPP。
 	// 配置值 <= Data算子输出边的个数。

parser/onnx/CMakeLists.txt

@@ -29,24 +29,12 @@ target_include_directories(fmk_onnx_parser PRIVATE
     ${PARSER_DIR}
     ${PARSER_DIR}/inc
     ${PARSER_DIR}/parser
-    ${PARSER_DIR}/../ge
-    ${PARSER_DIR}/../inc
-    ${PARSER_DIR}/../inc/common/util
-    ${PARSER_DIR}/../inc/framework
-    ${PARSER_DIR}/../inc/external
-    ${PARSER_DIR}/../third_party/fwkacllib/inc
     ${METADEF_DIR}/inc
     ${METADEF_DIR}/inc/graph
     ${METADEF_DIR}/inc/register
     ${METADEF_DIR}/inc/external
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/external/register
-    #### independent compile #####
-    ${METADEF_DIR}/third_party/graphengine/ge
-    ${METADEF_DIR}/third_party/graphengine/inc
-    ${METADEF_DIR}/third_party/graphengine/inc/framework
-    ${METADEF_DIR}/third_party/graphengine/inc/external
-    ${METADEF_DIR}/third_party/fwkacllib/inc
     #### temp ####
     ${PARSER_DIR}/../graphengine/inc/common/util
     ${PARSER_DIR}/../graphengine/inc/external
@@ -55,6 +43,19 @@ target_include_directories(fmk_onnx_parser PRIVATE
     ${PARSER_DIR}/../graphengine/ge
     ${CMAKE_BINARY_DIR}
     ${CMAKE_BINARY_DIR}/proto/ge
+    #### blue zone compile #####
+    ${PARSER_DIR}/../ge
+    ${PARSER_DIR}/../inc
+    ${PARSER_DIR}/../inc/common/util
+    ${PARSER_DIR}/../inc/framework
+    ${PARSER_DIR}/../inc/external
+    ${PARSER_DIR}/../third_party/fwkacllib/inc
+    #### independent compile #####
+    ${METADEF_DIR}/third_party/graphengine/ge
+    ${METADEF_DIR}/third_party/graphengine/inc
+    ${METADEF_DIR}/third_party/graphengine/inc/framework
+    ${METADEF_DIR}/third_party/graphengine/inc/external
+    ${METADEF_DIR}/third_party/fwkacllib/inc
 )
 
 target_link_libraries(fmk_onnx_parser PRIVATE

parser/proto/insert_op.proto

@@ -45,6 +45,9 @@ message AippOpParams {
 	// 标识对模型的第几个输入做AIPP处理，例如模型有两个输入，需要对第2个输入做AIPP，则配置related_input_rank为1。
 	uint32 related_input_rank = 2;
 
+        // related_input_name is optional and the top name of data node which inserts aipp
+        string related_input_name = 6;
+
 	// input_edge_idx参数为可选，类型为整型，配置范围为>=0。
 	// 配置该参数的作用，在于对Data算子不同的输出做不同的AIPP处理，如果该参数没有配置，默认对related_input_rank指定的模型输入的所有输出边做AIPP。
 	// 配置值 <= Data算子输出边的个数。

parser/tensorflow/proto/insert_op.proto

@@ -45,6 +45,9 @@ message AippOpParams {
 	// 标识对模型的第几个输入做AIPP处理，例如模型有两个输入，需要对第2个输入做AIPP，则配置related_input_rank为1。
 	uint32 related_input_rank = 2;
 
+        // related_input_name is optional and the top name of data node which inserts aipp
+        string related_input_name = 6;
+
 	// input_edge_idx参数为可选，类型为整型，配置范围为>=0。
 	// 配置该参数的作用，在于对Data算子不同的输出做不同的AIPP处理，如果该参数没有配置，默认对related_input_rank指定的模型输入的所有输出边做AIPP。
 	// 配置值 <= Data算子输出边的个数。