!24 sync from yellow zone 20201020

Merge pull request !24 from taoxiangdong/master

build.sh

@@ -120,12 +120,22 @@ build_parser()
   CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_OPEN_SRC=True -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH}"
   echo "${CMAKE_ARGS}"
   cmake ${CMAKE_ARGS} ..
+  if [ 0 -ne $? ]
+  then
+    echo "execute command: cmake ${CMAKE_ARGS} .. failed."
+    return 1
+  fi
   make ${VERBOSE} -j${THREAD_NUM} && make install
+  if [ 0 -ne $? ]
+  then
+    echo "execute command: make ${VERBOSE} -j${THREAD_NUM} && make install failed."
+    return 1
+  fi
   echo "Parser build success!"
 }
 g++ -v
 mk_dir ${OUTPUT_PATH}
-build_parser
+build_parser || { echo "Parser build failed."; return; }
 echo "---------------- Parser build finished ----------------"
 rm -f ${OUTPUT_PATH}/libgmock*.so
 rm -f ${OUTPUT_PATH}/libgtest*.so

cmake/external_libs/protobuf_shared.cmake

@@ -14,7 +14,7 @@ endif()
 set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
 set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
 ExternalProject_Add(protobuf_build
-	                URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
+                    URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
                     #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
 		            #SOURCE_DIR ${PARSER_DIR}/../third_party/protobuf/src/protobuf-3.8.0
 		            DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E copy_directory ${PARSER_DIR}/../third_party/protobuf/src/protobuf-3.8.0 <SOURCE_DIR>

cmake/external_libs/protoc.cmake

@@ -15,7 +15,7 @@ endif()
 set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
 set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
 ExternalProject_Add(protoc_build
-	                URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
+                    URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
                     #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
                     SOURCE_DIR ${PARSER_DIR}/../third_party/protobuf/src/protobuf-3.8.0
                     CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc <SOURCE_DIR>/cmake

metadef

@@ -1 +1 @@
-Subproject commit c599dfd701ad1f7a7ef149faedcf2b5dec93fafb
+Subproject commit 5d5df2e9f9c2a5a6f65f2fcb0136176298063199

parser/caffe/caffe_custom_parser_adapter.cc

@@ -83,7 +83,7 @@ Status CaffeCustomParserAdapter::ParseWeights(const Message *op_src, ge::NodePtr
   }
 
   bool bias_en = false;
-  bool update_in_turn = (static_cast<int64_t>(op->GetAllInputsSize()) == (layer->bottom_size() + layer->blobs_size()));
+  bool update_in_turn = (static_cast<int64_t >(op->GetAllInputsSize()) == (layer->bottom_size() + layer->blobs_size()));
   int start_pos = layer->bottom_size();
   for (int i = 0; i < layer->blobs_size(); ++i) {
     ge::GeTensorPtr weight = ge::parser::MakeShared<ge::GeTensor>();

parser/common/module.mk

@@ -73,9 +73,10 @@ LOCAL_C_INCLUDES := \
     $(TOPDIR)inc/external/graph \
     $(TOPDIR)inc/framework \
     $(TOPDIR)inc/common/util \
-    $(TOPDIR)framework/domi \
-    $(TOPDIR)framework/domi/common \
-    $(TOPDIR)framework/domi/parser \
+    $(TOPDIR)graphengine/ge \
+    $(TOPDIR)graphengine/ge/common \
+    $(TOPDIR)parser/parser \
+    $(TOPDIR)parser   \
     $(TOPDIR)third_party/json/include \
     $(TOPDIR)third_party/protobuf/include \
     libc_sec/include \

parser/common/proto/tensorflow/graph_library.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package domi.tensorflow;
+
+import "graph.proto";
+
+message GeGraphDef {
+  string name = 1;
+  GraphDef graph = 2;
+}
+
+message GraphDefLibrary {
+  repeated GeGraphDef graph_def = 1;
+};

parser/func_to_graph/proto_python_rule.mk

@@ -2,7 +2,7 @@ include $(BUILD_SYSTEM)/base_rules.mk
 
 FUNCTION_TO_GRAPH_OUT_TIMESTAMP := $(HOST_OUT_ROOT)/func_to_graph/.timestamp
 
-PROTO_SRC_DIR = framework/domi/parser/func_to_graph/proto
+PROTO_SRC_DIR = parser/parser/func_to_graph/proto
 PY_PROTO_BUILD_DIR = $(HOST_OUT_ROOT)/tmp/function_to_graph/proto
 
 $(warning PRIVATE_PROTOC is $(PRIVATE_PROTOC))
@@ -14,4 +14,4 @@ $(FUNCTION_TO_GRAPH_OUT_TIMESTAMP): $(PRIVATE_PROTOC)
 
 $(LOCAL_BUILT_MODULE): $(FUNCTION_TO_GRAPH_OUT_TIMESTAMP)
 	mkdir -p $@
-	cp -rf $(PY_PROTO_BUILD_DIR)/* $@
+	cp -rf $(PY_PROTO_BUILD_DIR)/* $@

parser/module.mk

@@ -1,6 +1,6 @@
 
 LOCAL_PATH := $(call my-dir)
-include $(LOCAL_PATH)/../stub/Makefile
+include $(LOCAL_PATH)/stub/Makefile
 COMMON_LOCAL_C_INCLUDES := \
     proto/om.proto \
     proto/insert_op.proto \
@@ -39,7 +39,9 @@ COMMON_LOCAL_C_INCLUDES := \
     $(TOPDIR)inc/external/graph \
     $(TOPDIR)inc/external/parser \
     $(TOPDIR)inc/framework \
-    $(TOPDIR)framework/domi/parser \
+    $(TOPDIR)parser/parser \
+    $(TOPDIR)parser        \
+    $(TOPDIR)graphengine/ge \
     libc_sec/include \
     third_party/protobuf/include \
     third_party/json/include \
@@ -132,8 +134,8 @@ endif
 
 LOCAL_C_INCLUDES := $(COMMON_LOCAL_C_INCLUDES)
 
-LOCAL_SRC_FILES := ../../../out/ge/lib64/stub/tensorflow_parser.cc
-LOCAL_SRC_FILES += ../../../out/ge/lib64/stub/caffe_parser.cc
+LOCAL_SRC_FILES := ../../out/parser/lib64/stub/tensorflow_parser.cc
+LOCAL_SRC_FILES += ../../out/parser/lib64/stub/caffe_parser.cc
 
 
 LOCAL_SHARED_LIBRARIES :=

parser/onnx/module.mk

@@ -29,7 +29,9 @@ LOCAL_C_INCLUDES := \
     $(TOPDIR)inc/external \
     $(TOPDIR)inc/external/graph \
     $(TOPDIR)inc/framework \
-    $(TOPDIR)framework/domi/parser \
+    $(TOPDIR)parser \
+    $(TOPDIR)parser/parser \
+    $(TOPDIR)graphengine/ge \
     libc_sec/include \
     third_party/protobuf/include \
     third_party/json/include \

parser/ops/op_imp.cpp

@@ -1,63 +0,0 @@
-#include <stdint.h>
-#include <functional>
-#include <vector>
-#include "debug/ge_log.h"
-#include "debug/ge_util.h"
-
-using namespace std;
-
-namespace ge {
-
-GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY graphStatus
-BroadCastInfer(const function<vector<int64_t>()>& get_in1_shape, const function<vector<int64_t>()>& get_in2_shape,
-               const function<void(const vector<int64_t>& outShape)>& set_out_shape) {
-  auto x1_shape = get_in1_shape();
-  auto x2_shape = get_in2_shape();
-  vector<int64_t> y_shape;
-
-  if (x1_shape.empty()) {
-    y_shape = x2_shape;
-    set_out_shape(y_shape);
-    return GRAPH_SUCCESS;
-  }
-  if (x2_shape.empty()) {
-    y_shape = x1_shape;
-    set_out_shape(y_shape);
-    return GRAPH_SUCCESS;
-  }
-
-  int len_diff = static_cast<int>(x1_shape.size() - x2_shape.size());
-  if (len_diff >= 0) {
-    for (int i = 0; i < len_diff; i++) {
-      y_shape.push_back(x1_shape[i]);
-    }
-    int x2_shape_size = static_cast<int>(x2_shape.size());
-    for (int i = 0; i < x2_shape_size; i++) {
-      bool shapeFlag =
-          ((x1_shape[i + len_diff] != x2_shape[i]) && (std::min(x1_shape[i + len_diff], x2_shape[i]) != 1));
-      if (shapeFlag) {
-        GE_LOGE("operands could not be broadcast together");
-        return GRAPH_FAILED;
-      }
-      y_shape.push_back(std::max(x1_shape[i + len_diff], x2_shape[i]));
-    }
-  } else {
-    for (int i = 0; i < -len_diff; i++) {
-      y_shape.push_back(x2_shape[i]);
-    }
-    int x1_shape_size = static_cast<int>(x1_shape.size());
-    for (int i = 0; i < x1_shape_size; i++) {
-      bool shapeFlag =
-          ((x1_shape[i] != x2_shape[i - len_diff]) && (std::min(x1_shape[i], x2_shape[i - len_diff]) != 1));
-      if (shapeFlag) {
-        GE_LOGE("operands could not be broadcast together");
-        return GRAPH_FAILED;
-      }
-      y_shape.push_back(std::max(x1_shape[i], x2_shape[i - len_diff]));
-    }
-  }
-  set_out_shape(y_shape);
-  return GRAPH_SUCCESS;
-}
-
-}  // namespace ge

parser/stub/Makefile

@@ -1,6 +1,6 @@
-inc_path := $(shell pwd)/inc/external/
-out_path := $(shell pwd)/out/graph/lib64/stub/
-stub_path := $(shell pwd)/common/graph/stub/
+inc_path := $(shell pwd)/parser/inc/external/
+out_path := $(shell pwd)/out/parser/lib64/stub/
+stub_path := $(shell pwd)/parser/parser/stub/
 
 mkdir_stub := $(shell mkdir -p $(out_path))
-graph_local_stub := $(shell $(HI_PYTHON) $(stub_path)/gen_stubapi.py $(inc_path) $(out_path))
+local_stub := $(shell $(HI_PYTHON) $(stub_path)/gen_stubapi.py $(inc_path) $(out_path))

parser/stub/README

@@ -0,0 +1,4 @@
+###################################################################################
+the directory (stub) saves the stub file
+gen_stubapi.py is using for retrieving API and generating stub functions
+###################################################################################

parser/stub/README.md

@@ -0,0 +1,44 @@
+# "stub"  usage:
+
+## Description
+
+- File libge_compiler.so ,libgraph.so are used in IR build application interface.
+
+# Attention
+
+- Don't link other library except libge_compiler.so ,libgraph.so, as they may be changed in the future.
+
+# Usage
+
+## Compile:   compile  the application invoking the IR build API.
+
+Makefile:
+
+'''
+
+ATC_INCLUDE_DIR := $(ASCEND_PATH)/atc/include
+OPP_INCLUDE_DIR := $(ASCEND_PATH)/opp/op_proto/built-in/inc
+LOCAL_MODULE_NAME := ir_build
+CC := g++
+CFLAGS := -std=c++11 -g -Wall
+SRCS := $(wildcard $(LOCAL_DIR)/main.cpp)
+INCLUDES := -I $(ASCEND_OPP_PATH)/op_proto/built-in/inc \
+            -I $(ATC_INCLUDE_DIR)/graph \
+            -I $(ATC_INCLUDE_DIR)/ge \
+
+LIBS := -L ${ASCEND_PATH}/atc/lib64/stub \
+    -lgraph \
+    -lge_compiler
+ir_build:
+    mkdir -p out
+    $(CC) $(SRCS) $(INCLUDES) $(LIBS) $(CFLAGS) -o ./out/$(LOCAL_MODULE_NAME)
+clean:
+    rm -rf out
+
+'''
+make
+
+## Run the application after set the LD_LIBRARY_PATH to include the real path of the library which locates in the directory of atc/lib64
+
+export LD_LIBRARY_PATH= $(ASCEND_PATH)/atc/lib64
+ -  ./ ir_build

parser/stub/gen_stubapi.py

@@ -63,8 +63,9 @@ max_code_len_per_line = 100
     determines which header files to generate cc files from
     when DEBUG on
 """
-white_list_for_debug = ["tensorflow_parser.h", "caffe_parser.h"]
-include_dir_key_words = ["parser"]
+white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h",
+                        "ge_ir_build.h", "ge_api.h", "ge_prof.h", "tensorflow_parser.h", "caffe_parser.h"]
+include_dir_key_words = ["ge", "graph", "parser"]
 DEBUG = True
 
 
@@ -101,7 +102,7 @@ pattern_func = re.compile(r"""(^[\s]*)          #leading with space,we will find
 ([a-zA-Z~_]            # void int likely
 .*
 [)]                     #we find )
-(?!.*{)                 # we do not want the case int abc() const
+(?!.*{)                 # we do not want the case int abc() const { return 1;}
 .*)
 (;.*)                   #we want to find ; and after for we will replace these later
 \n$