diff --git a/build.sh b/build.sh index 75ffa39..655cd78 100644 --- a/build.sh +++ b/build.sh @@ -120,12 +120,22 @@ build_parser() CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_OPEN_SRC=True -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH}" echo "${CMAKE_ARGS}" cmake ${CMAKE_ARGS} .. + if [ 0 -ne $? ] + then + echo "execute command: cmake ${CMAKE_ARGS} .. failed." + return 1 + fi make ${VERBOSE} -j${THREAD_NUM} && make install + if [ 0 -ne $? ] + then + echo "execute command: make ${VERBOSE} -j${THREAD_NUM} && make install failed." + return 1 + fi echo "Parser build success!" } g++ -v mk_dir ${OUTPUT_PATH} -build_parser +build_parser || { echo "Parser build failed."; return; } echo "---------------- Parser build finished ----------------" rm -f ${OUTPUT_PATH}/libgmock*.so rm -f ${OUTPUT_PATH}/libgtest*.so diff --git a/cmake/external_libs/protobuf_shared.cmake b/cmake/external_libs/protobuf_shared.cmake index 0581ae8..265b309 100755 --- a/cmake/external_libs/protobuf_shared.cmake +++ b/cmake/external_libs/protobuf_shared.cmake @@ -14,7 +14,7 @@ endif() set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") ExternalProject_Add(protobuf_build - URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz + URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz #SOURCE_DIR ${PARSER_DIR}/../third_party/protobuf/src/protobuf-3.8.0 DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E copy_directory ${PARSER_DIR}/../third_party/protobuf/src/protobuf-3.8.0 diff --git a/cmake/external_libs/protoc.cmake b/cmake/external_libs/protoc.cmake index ff49681..9ec86e2 100755 --- a/cmake/external_libs/protoc.cmake +++ b/cmake/external_libs/protoc.cmake @@ -15,7 +15,7 @@ endif() set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") ExternalProject_Add(protoc_build - URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz + URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz SOURCE_DIR ${PARSER_DIR}/../third_party/protobuf/src/protobuf-3.8.0 CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc /cmake diff --git a/metadef b/metadef index c599dfd..5d5df2e 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit c599dfd701ad1f7a7ef149faedcf2b5dec93fafb +Subproject commit 5d5df2e9f9c2a5a6f65f2fcb0136176298063199 diff --git a/parser/caffe/caffe_custom_parser_adapter.cc b/parser/caffe/caffe_custom_parser_adapter.cc index 8b18553..d250f38 100644 --- a/parser/caffe/caffe_custom_parser_adapter.cc +++ b/parser/caffe/caffe_custom_parser_adapter.cc @@ -83,7 +83,7 @@ Status CaffeCustomParserAdapter::ParseWeights(const Message *op_src, ge::NodePtr } bool bias_en = false; - bool update_in_turn = (static_cast(op->GetAllInputsSize()) == (layer->bottom_size() + layer->blobs_size())); + bool update_in_turn = (static_cast(op->GetAllInputsSize()) == (layer->bottom_size() + layer->blobs_size())); int start_pos = layer->bottom_size(); for (int i = 0; i < layer->blobs_size(); ++i) { ge::GeTensorPtr weight = ge::parser::MakeShared(); diff --git a/parser/common/module.mk b/parser/common/module.mk index 5a567c0..4eb3e74 100644 --- a/parser/common/module.mk +++ b/parser/common/module.mk @@ -73,9 +73,10 @@ LOCAL_C_INCLUDES := \ $(TOPDIR)inc/external/graph \ $(TOPDIR)inc/framework \ $(TOPDIR)inc/common/util \ - $(TOPDIR)framework/domi \ - $(TOPDIR)framework/domi/common \ - $(TOPDIR)framework/domi/parser \ + $(TOPDIR)graphengine/ge \ + $(TOPDIR)graphengine/ge/common \ + $(TOPDIR)parser/parser \ + $(TOPDIR)parser \ $(TOPDIR)third_party/json/include \ $(TOPDIR)third_party/protobuf/include \ libc_sec/include \ diff --git a/parser/common/proto/tensorflow/graph_library.proto b/parser/common/proto/tensorflow/graph_library.proto new file mode 100644 index 0000000..e393d38 --- /dev/null +++ b/parser/common/proto/tensorflow/graph_library.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package domi.tensorflow; + +import "graph.proto"; + +message GeGraphDef { + string name = 1; + GraphDef graph = 2; +} + +message GraphDefLibrary { + repeated GeGraphDef graph_def = 1; +}; \ No newline at end of file diff --git a/parser/func_to_graph/proto_python_rule.mk b/parser/func_to_graph/proto_python_rule.mk index ece9732..cf442d5 100644 --- a/parser/func_to_graph/proto_python_rule.mk +++ b/parser/func_to_graph/proto_python_rule.mk @@ -2,7 +2,7 @@ include $(BUILD_SYSTEM)/base_rules.mk FUNCTION_TO_GRAPH_OUT_TIMESTAMP := $(HOST_OUT_ROOT)/func_to_graph/.timestamp -PROTO_SRC_DIR = framework/domi/parser/func_to_graph/proto +PROTO_SRC_DIR = parser/parser/func_to_graph/proto PY_PROTO_BUILD_DIR = $(HOST_OUT_ROOT)/tmp/function_to_graph/proto $(warning PRIVATE_PROTOC is $(PRIVATE_PROTOC)) @@ -14,4 +14,4 @@ $(FUNCTION_TO_GRAPH_OUT_TIMESTAMP): $(PRIVATE_PROTOC) $(LOCAL_BUILT_MODULE): $(FUNCTION_TO_GRAPH_OUT_TIMESTAMP) mkdir -p $@ - cp -rf $(PY_PROTO_BUILD_DIR)/* $@ \ No newline at end of file + cp -rf $(PY_PROTO_BUILD_DIR)/* $@ diff --git a/parser/module.mk b/parser/module.mk index 319e7a6..502d44b 100644 --- a/parser/module.mk +++ b/parser/module.mk @@ -1,6 +1,6 @@ LOCAL_PATH := $(call my-dir) -include $(LOCAL_PATH)/../stub/Makefile +include $(LOCAL_PATH)/stub/Makefile COMMON_LOCAL_C_INCLUDES := \ proto/om.proto \ proto/insert_op.proto \ @@ -39,7 +39,9 @@ COMMON_LOCAL_C_INCLUDES := \ $(TOPDIR)inc/external/graph \ $(TOPDIR)inc/external/parser \ $(TOPDIR)inc/framework \ - $(TOPDIR)framework/domi/parser \ + $(TOPDIR)parser/parser \ + $(TOPDIR)parser \ + $(TOPDIR)graphengine/ge \ libc_sec/include \ third_party/protobuf/include \ third_party/json/include \ @@ -132,8 +134,8 @@ endif LOCAL_C_INCLUDES := $(COMMON_LOCAL_C_INCLUDES) -LOCAL_SRC_FILES := ../../../out/ge/lib64/stub/tensorflow_parser.cc -LOCAL_SRC_FILES += ../../../out/ge/lib64/stub/caffe_parser.cc +LOCAL_SRC_FILES := ../../out/parser/lib64/stub/tensorflow_parser.cc +LOCAL_SRC_FILES += ../../out/parser/lib64/stub/caffe_parser.cc LOCAL_SHARED_LIBRARIES := diff --git a/parser/onnx/module.mk b/parser/onnx/module.mk index 394c369..531be98 100644 --- a/parser/onnx/module.mk +++ b/parser/onnx/module.mk @@ -29,7 +29,9 @@ LOCAL_C_INCLUDES := \ $(TOPDIR)inc/external \ $(TOPDIR)inc/external/graph \ $(TOPDIR)inc/framework \ - $(TOPDIR)framework/domi/parser \ + $(TOPDIR)parser \ + $(TOPDIR)parser/parser \ + $(TOPDIR)graphengine/ge \ libc_sec/include \ third_party/protobuf/include \ third_party/json/include \ diff --git a/parser/ops/op_imp.cpp b/parser/ops/op_imp.cpp deleted file mode 100644 index 4f8e1de..0000000 --- a/parser/ops/op_imp.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include -#include -#include -#include "debug/ge_log.h" -#include "debug/ge_util.h" - -using namespace std; - -namespace ge { - -GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY graphStatus -BroadCastInfer(const function()>& get_in1_shape, const function()>& get_in2_shape, - const function& outShape)>& set_out_shape) { - auto x1_shape = get_in1_shape(); - auto x2_shape = get_in2_shape(); - vector y_shape; - - if (x1_shape.empty()) { - y_shape = x2_shape; - set_out_shape(y_shape); - return GRAPH_SUCCESS; - } - if (x2_shape.empty()) { - y_shape = x1_shape; - set_out_shape(y_shape); - return GRAPH_SUCCESS; - } - - int len_diff = static_cast(x1_shape.size() - x2_shape.size()); - if (len_diff >= 0) { - for (int i = 0; i < len_diff; i++) { - y_shape.push_back(x1_shape[i]); - } - int x2_shape_size = static_cast(x2_shape.size()); - for (int i = 0; i < x2_shape_size; i++) { - bool shapeFlag = - ((x1_shape[i + len_diff] != x2_shape[i]) && (std::min(x1_shape[i + len_diff], x2_shape[i]) != 1)); - if (shapeFlag) { - GE_LOGE("operands could not be broadcast together"); - return GRAPH_FAILED; - } - y_shape.push_back(std::max(x1_shape[i + len_diff], x2_shape[i])); - } - } else { - for (int i = 0; i < -len_diff; i++) { - y_shape.push_back(x2_shape[i]); - } - int x1_shape_size = static_cast(x1_shape.size()); - for (int i = 0; i < x1_shape_size; i++) { - bool shapeFlag = - ((x1_shape[i] != x2_shape[i - len_diff]) && (std::min(x1_shape[i], x2_shape[i - len_diff]) != 1)); - if (shapeFlag) { - GE_LOGE("operands could not be broadcast together"); - return GRAPH_FAILED; - } - y_shape.push_back(std::max(x1_shape[i], x2_shape[i - len_diff])); - } - } - set_out_shape(y_shape); - return GRAPH_SUCCESS; -} - -} // namespace ge diff --git a/parser/stub/Makefile b/parser/stub/Makefile index 520b292..51b4651 100644 --- a/parser/stub/Makefile +++ b/parser/stub/Makefile @@ -1,6 +1,6 @@ -inc_path := $(shell pwd)/inc/external/ -out_path := $(shell pwd)/out/graph/lib64/stub/ -stub_path := $(shell pwd)/common/graph/stub/ +inc_path := $(shell pwd)/parser/inc/external/ +out_path := $(shell pwd)/out/parser/lib64/stub/ +stub_path := $(shell pwd)/parser/parser/stub/ mkdir_stub := $(shell mkdir -p $(out_path)) -graph_local_stub := $(shell $(HI_PYTHON) $(stub_path)/gen_stubapi.py $(inc_path) $(out_path)) +local_stub := $(shell $(HI_PYTHON) $(stub_path)/gen_stubapi.py $(inc_path) $(out_path)) diff --git a/parser/stub/README b/parser/stub/README new file mode 100644 index 0000000..ca98ce8 --- /dev/null +++ b/parser/stub/README @@ -0,0 +1,4 @@ +################################################################################### +the directory (stub) saves the stub file +gen_stubapi.py is using for retrieving API and generating stub functions +################################################################################### diff --git a/parser/stub/README.md b/parser/stub/README.md new file mode 100644 index 0000000..a085e53 --- /dev/null +++ b/parser/stub/README.md @@ -0,0 +1,44 @@ +# "stub" usage: + +## Description + +- File libge_compiler.so ,libgraph.so are used in IR build application interface. + +# Attention + +- Don't link other library except libge_compiler.so ,libgraph.so, as they may be changed in the future. + +# Usage + +## Compile: compile the application invoking the IR build API. + +Makefile: + +''' + +ATC_INCLUDE_DIR := $(ASCEND_PATH)/atc/include +OPP_INCLUDE_DIR := $(ASCEND_PATH)/opp/op_proto/built-in/inc +LOCAL_MODULE_NAME := ir_build +CC := g++ +CFLAGS := -std=c++11 -g -Wall +SRCS := $(wildcard $(LOCAL_DIR)/main.cpp) +INCLUDES := -I $(ASCEND_OPP_PATH)/op_proto/built-in/inc \ + -I $(ATC_INCLUDE_DIR)/graph \ + -I $(ATC_INCLUDE_DIR)/ge \ + +LIBS := -L ${ASCEND_PATH}/atc/lib64/stub \ + -lgraph \ + -lge_compiler +ir_build: + mkdir -p out + $(CC) $(SRCS) $(INCLUDES) $(LIBS) $(CFLAGS) -o ./out/$(LOCAL_MODULE_NAME) +clean: + rm -rf out + +''' +make + +## Run the application after set the LD_LIBRARY_PATH to include the real path of the library which locates in the directory of atc/lib64 + +export LD_LIBRARY_PATH= $(ASCEND_PATH)/atc/lib64 + - ./ ir_build diff --git a/parser/stub/gen_stubapi.py b/parser/stub/gen_stubapi.py index 3e2c51c..0c5e712 100644 --- a/parser/stub/gen_stubapi.py +++ b/parser/stub/gen_stubapi.py @@ -63,8 +63,9 @@ max_code_len_per_line = 100 determines which header files to generate cc files from when DEBUG on """ -white_list_for_debug = ["tensorflow_parser.h", "caffe_parser.h"] -include_dir_key_words = ["parser"] +white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h", + "ge_ir_build.h", "ge_api.h", "ge_prof.h", "tensorflow_parser.h", "caffe_parser.h"] +include_dir_key_words = ["ge", "graph", "parser"] DEBUG = True @@ -101,7 +102,7 @@ pattern_func = re.compile(r"""(^[\s]*) #leading with space,we will find ([a-zA-Z~_] # void int likely .* [)] #we find ) -(?!.*{) # we do not want the case int abc() const +(?!.*{) # we do not want the case int abc() const { return 1;} .*) (;.*) #we want to find ; and after for we will replace these later \n$