diff --git a/.gitmodules b/.gitmodules index fe72c1e..03812e0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "metadef"] path = metadef url = https://gitee.com/ascend/metadef.git + branch = development diff --git a/build.sh b/build.sh index 655cd78..f3a54b3 100644 --- a/build.sh +++ b/build.sh @@ -86,6 +86,8 @@ checkopts() } checkopts "$@" +git submodule update --init metadef + mk_dir() { local create_dir="$1" # the target to make @@ -152,12 +154,41 @@ generate_package() cd "${BASEPATH}" PARSER_LIB_PATH="lib" + ACL_PATH="acllib/lib64" + FWK_PATH="fwkacllib/lib64" + ATC_PATH="atc/lib64" + + COMMON_LIB=("libgraph.so" "libregister.so") + PARSER_LIB=("lib_caffe_parser.so" "libfmk_onnx_parser.so" "libfmk_parser.so" "libparser_common.so") + + rm -rf ${OUTPUT_PATH:?}/${FWK_PATH}/ + rm -rf ${OUTPUT_PATH:?}/${ACL_PATH}/ + rm -rf ${OUTPUT_PATH:?}/${ATC_PATH}/ + + mk_dir "${OUTPUT_PATH}/${FWK_PATH}" + mk_dir "${OUTPUT_PATH}/${ATC_PATH}" + mk_dir "${OUTPUT_PATH}/${ACL_PATH}" find output/ -name parser_lib.tar -exec rm {} \; cd "${OUTPUT_PATH}" - tar -cf parser_lib.tar "${PARSER_LIB_PATH}" + for lib in "${PARSER_LIB[@]}"; + do + find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH} \; + find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \; + done + + for lib in "${COMMON_LIB[@]}"; + do + find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH} \; + find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \; + done + + find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "libc_sec.so" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \; + find ${OUTPUT_PATH}/${PARSER_LIB_PATH} -maxdepth 1 -name "libregister.a" -exec cp -f {} ${OUTPUT_PATH}/${ACL_PATH} \; + + tar -cf parser_lib.tar fwkacllib acllib atc } if [[ "X$ENABLE_GE_UT" = "Xoff" ]]; then diff --git a/metadef b/metadef index 765d857..cc9de48 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit 765d85777ec10fe819799cd2c1b60db49be7c749 +Subproject commit cc9de48a7779cf95cab90a23db608421a691fd12 diff --git a/parser/CMakeLists.txt b/parser/CMakeLists.txt index 3b17728..db446e1 100644 --- a/parser/CMakeLists.txt +++ b/parser/CMakeLists.txt @@ -59,24 +59,12 @@ target_include_directories(fmk_parser PRIVATE ${PARSER_DIR} ${PARSER_DIR}/inc ${PARSER_DIR}/parser - ${PARSER_DIR}/../ge - ${PARSER_DIR}/../inc - ${PARSER_DIR}/../inc/framework - ${PARSER_DIR}/../inc/common/util - ${PARSER_DIR}/../inc/external - ${PARSER_DIR}/../third_party/fwkacllib/inc ${METADEF_DIR}/inc ${METADEF_DIR}/inc/graph ${METADEF_DIR}/inc/register ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/external/register - #### independent compile ##### - ${METADEF_DIR}/third_party/graphengine/ge - ${METADEF_DIR}/third_party/graphengine/inc - ${METADEF_DIR}/third_party/graphengine/inc/framework - ${METADEF_DIR}/third_party/graphengine/inc/external - ${METADEF_DIR}/third_party/fwkacllib/inc #### temp #### ${PARSER_DIR}/../graphengine/inc/common/util ${PARSER_DIR}/../graphengine/inc/external @@ -84,7 +72,20 @@ target_include_directories(fmk_parser PRIVATE ${PARSER_DIR}/../graphengine/inc ${PARSER_DIR}/../graphengine/ge ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/ge + #### blue zone compile ##### + ${PARSER_DIR}/../ge + ${PARSER_DIR}/../inc + ${PARSER_DIR}/../inc/framework + ${PARSER_DIR}/../inc/common/util + ${PARSER_DIR}/../inc/external + ${PARSER_DIR}/../third_party/fwkacllib/inc + #### blue independent compile ##### + ${METADEF_DIR}/third_party/graphengine/ge + ${METADEF_DIR}/third_party/graphengine/inc + ${METADEF_DIR}/third_party/graphengine/inc/framework + ${METADEF_DIR}/third_party/graphengine/inc/external + ${METADEF_DIR}/third_party/fwkacllib/inc ) target_link_libraries(fmk_parser diff --git a/parser/caffe/caffe_parser.cc b/parser/caffe/caffe_parser.cc index 36d07b3..4b3b2a7 100644 --- a/parser/caffe/caffe_parser.cc +++ b/parser/caffe/caffe_parser.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "parser/common/convert/pb2json.h" #include "common/debug/log.h" #include "parser/common/acl_graph_parser_util.h" @@ -1539,6 +1540,34 @@ void CaffeModelParser::SaveOrigionLayerTops(domi::caffe::LayerParameter &layer) return; } +Status CaffeModelParser::SaveDataLayerTops(const domi::caffe::LayerParameter &layer) { + string name = layer.name(); + if (node_map.find(name) == node_map.end()) { + GELOGE(FAILED, "Node can not be found by layer name: %s", name.c_str()); + return FAILED; + } + + ge::NodePtr node = node_map[name]; + GE_CHECK_NOTNULL(node); + + if (node->GetType() == ge::parser::DATA) { + if (layer.top_size() != 1) { + GELOGE(FAILED, "Data layer[%s] top size must be 1, real size: %d", name.c_str(), layer.top_size()); + return FAILED; + } + + string top_name = layer.top(0); + auto data_top_names = ge::GetParserContext().data_top_names; + if (find(data_top_names.begin(), data_top_names.end(), top_name) != data_top_names.end()) { + GELOGE(FAILED, "Different data can not have same top name: %s.", top_name.c_str()); + return FAILED; + } + ge::GetParserContext().data_top_names.push_back(top_name); + } + + return SUCCESS; +} + Status CaffeModelParser::Parse(const char *model_path, ge::ComputeGraphPtr &graph) { bool has_error = false; GE_CHECK_NOTNULL(model_path); @@ -1610,6 +1639,7 @@ Status CaffeModelParser::Parse(const char *model_path, ge::ComputeGraphPtr &grap // Map of operator name and occurrence times std::map layer_name_map; + GetParserContext().data_top_names.clear(); // std::map> layer_params_map; // same param name set @@ -1656,6 +1686,10 @@ Status CaffeModelParser::Parse(const char *model_path, ge::ComputeGraphPtr &grap GE_RETURN_WITH_LOG_IF_ERROR(AddBlobsToMap(layer, inplace_blob_name_remapping), "Caffe parser add blobs to map ret fail."); + if (SaveDataLayerTops(layer) != SUCCESS) { + GELOGE(FAILED, "Caffe parse: save data layer tops failed."); + return FAILED; + } } // Find a layer with the same param name and save it to graph GE_RETURN_WITH_LOG_IF_ERROR(FindShareParamLayers(layer_params_map), diff --git a/parser/caffe/caffe_parser.h b/parser/caffe/caffe_parser.h index bd1520e..9a252f9 100644 --- a/parser/caffe/caffe_parser.h +++ b/parser/caffe/caffe_parser.h @@ -343,6 +343,8 @@ class CaffeModelParser : public domi::ModelParser { Status ParseOutputNodeTopInfo(const domi::caffe::NetParameter &proto_message); + Status SaveDataLayerTops(const domi::caffe::LayerParameter &layer); + std::map node_map; // key: blob name, value: layer name and index diff --git a/parser/common/CMakeLists.txt b/parser/common/CMakeLists.txt index 9c3be13..276d2c2 100644 --- a/parser/common/CMakeLists.txt +++ b/parser/common/CMakeLists.txt @@ -41,24 +41,12 @@ target_include_directories(parser_common PRIVATE ${CMAKE_CURRENT_LIST_DIR} ${PARSER_DIR} ${PARSER_DIR}/parser - ${PARSER_DIR}/../ge - ${PARSER_DIR}/../inc - ${PARSER_DIR}/../inc/framework - ${PARSER_DIR}/../inc/common/util - ${PARSER_DIR}/../inc/external - ${PARSER_DIR}/../third_party/fwkacllib/inc ${METADEF_DIR}/inc ${METADEF_DIR}/inc/graph ${METADEF_DIR}/inc/register ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/external/register - #### independent compile ##### - ${METADEF_DIR}/third_party/graphengine/ge - ${METADEF_DIR}/third_party/graphengine/inc - ${METADEF_DIR}/third_party/graphengine/inc/framework - ${METADEF_DIR}/third_party/graphengine/inc/external - ${METADEF_DIR}/third_party/fwkacllib/inc #### temp #### ${PARSER_DIR}/../graphengine/inc/common/util ${PARSER_DIR}/../graphengine/inc/external @@ -67,6 +55,19 @@ target_include_directories(parser_common PRIVATE ${PARSER_DIR}/../graphengine/ge ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/proto/ge + #### blue zone compile ##### + ${PARSER_DIR}/../ge + ${PARSER_DIR}/../inc + ${PARSER_DIR}/../inc/framework + ${PARSER_DIR}/../inc/common/util + ${PARSER_DIR}/../inc/external + ${PARSER_DIR}/../third_party/fwkacllib/inc + #### independent compile ##### + ${METADEF_DIR}/third_party/graphengine/ge + ${METADEF_DIR}/third_party/graphengine/inc + ${METADEF_DIR}/third_party/graphengine/inc/framework + ${METADEF_DIR}/third_party/graphengine/inc/external + ${METADEF_DIR}/third_party/fwkacllib/inc ) target_link_libraries(parser_common PRIVATE diff --git a/parser/common/proto/insert_op.proto b/parser/common/proto/insert_op.proto index e28d653..6c3287a 100644 --- a/parser/common/proto/insert_op.proto +++ b/parser/common/proto/insert_op.proto @@ -45,6 +45,9 @@ message AippOpParams { // 标识对模型的第几个输入做AIPP处理,例如模型有两个输入,需要对第2个输入做AIPP,则配置related_input_rank为1。 uint32 related_input_rank = 2; + // related_input_name is optional and the top name of data node which inserts aipp + string related_input_name = 6; + // input_edge_idx参数为可选,类型为整型,配置范围为>=0。 // 配置该参数的作用,在于对Data算子不同的输出做不同的AIPP处理,如果该参数没有配置,默认对related_input_rank指定的模型输入的所有输出边做AIPP。 // 配置值 <= Data算子输出边的个数。 diff --git a/parser/onnx/CMakeLists.txt b/parser/onnx/CMakeLists.txt index 26fcce4..84d1178 100644 --- a/parser/onnx/CMakeLists.txt +++ b/parser/onnx/CMakeLists.txt @@ -29,24 +29,12 @@ target_include_directories(fmk_onnx_parser PRIVATE ${PARSER_DIR} ${PARSER_DIR}/inc ${PARSER_DIR}/parser - ${PARSER_DIR}/../ge - ${PARSER_DIR}/../inc - ${PARSER_DIR}/../inc/common/util - ${PARSER_DIR}/../inc/framework - ${PARSER_DIR}/../inc/external - ${PARSER_DIR}/../third_party/fwkacllib/inc ${METADEF_DIR}/inc ${METADEF_DIR}/inc/graph ${METADEF_DIR}/inc/register ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/external/register - #### independent compile ##### - ${METADEF_DIR}/third_party/graphengine/ge - ${METADEF_DIR}/third_party/graphengine/inc - ${METADEF_DIR}/third_party/graphengine/inc/framework - ${METADEF_DIR}/third_party/graphengine/inc/external - ${METADEF_DIR}/third_party/fwkacllib/inc #### temp #### ${PARSER_DIR}/../graphengine/inc/common/util ${PARSER_DIR}/../graphengine/inc/external @@ -55,6 +43,19 @@ target_include_directories(fmk_onnx_parser PRIVATE ${PARSER_DIR}/../graphengine/ge ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/proto/ge + #### blue zone compile ##### + ${PARSER_DIR}/../ge + ${PARSER_DIR}/../inc + ${PARSER_DIR}/../inc/common/util + ${PARSER_DIR}/../inc/framework + ${PARSER_DIR}/../inc/external + ${PARSER_DIR}/../third_party/fwkacllib/inc + #### independent compile ##### + ${METADEF_DIR}/third_party/graphengine/ge + ${METADEF_DIR}/third_party/graphengine/inc + ${METADEF_DIR}/third_party/graphengine/inc/framework + ${METADEF_DIR}/third_party/graphengine/inc/external + ${METADEF_DIR}/third_party/fwkacllib/inc ) target_link_libraries(fmk_onnx_parser PRIVATE diff --git a/parser/proto/insert_op.proto b/parser/proto/insert_op.proto index e28d653..6c3287a 100644 --- a/parser/proto/insert_op.proto +++ b/parser/proto/insert_op.proto @@ -45,6 +45,9 @@ message AippOpParams { // 标识对模型的第几个输入做AIPP处理,例如模型有两个输入,需要对第2个输入做AIPP,则配置related_input_rank为1。 uint32 related_input_rank = 2; + // related_input_name is optional and the top name of data node which inserts aipp + string related_input_name = 6; + // input_edge_idx参数为可选,类型为整型,配置范围为>=0。 // 配置该参数的作用,在于对Data算子不同的输出做不同的AIPP处理,如果该参数没有配置,默认对related_input_rank指定的模型输入的所有输出边做AIPP。 // 配置值 <= Data算子输出边的个数。 diff --git a/parser/tensorflow/proto/insert_op.proto b/parser/tensorflow/proto/insert_op.proto index e28d653..6c3287a 100644 --- a/parser/tensorflow/proto/insert_op.proto +++ b/parser/tensorflow/proto/insert_op.proto @@ -45,6 +45,9 @@ message AippOpParams { // 标识对模型的第几个输入做AIPP处理,例如模型有两个输入,需要对第2个输入做AIPP,则配置related_input_rank为1。 uint32 related_input_rank = 2; + // related_input_name is optional and the top name of data node which inserts aipp + string related_input_name = 6; + // input_edge_idx参数为可选,类型为整型,配置范围为>=0。 // 配置该参数的作用,在于对Data算子不同的输出做不同的AIPP处理,如果该参数没有配置,默认对related_input_rank指定的模型输入的所有输出边做AIPP。 // 配置值 <= Data算子输出边的个数。