：serving 0.1.0

.clang-format

@@ -0,0 +1,151 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 2
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+#  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^<ext/.*\.h>'
+    Priority:        2
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+RawStringFormats:
+  - Language:        Cpp
+    Delimiters:
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+  - Language:        TextProto
+    Delimiters:
+      - pb
+      - PB
+      - proto
+      - PROTO
+    EnclosingFunctions:
+      - EqualsProto
+      - EquivToProto
+      - PARSE_PARTIAL_TEXT_PROTO
+      - PARSE_TEST_PROTO
+      - PARSE_TEXT_PROTO
+      - ParseTextOrDie
+      - ParseTextProtoOrDie
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+ReflowComments:  true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        2
+UseTab:          Never
+SortIncludes:    false
+...
+

.gitignore

@@ -1,3 +1,16 @@
+# MindSpore Serving
+build/
+mindspore_serving/lib
+output
+*.ir
+
+# Cmake files
+CMakeFiles/
+cmake_install.cmake
+CMakeCache.txt
+Makefile
+cmake-build-debug
+
 # Prerequisites
 *.d
 
@@ -15,6 +28,7 @@
 *.so
 *.dylib
 *.dll
+*.so.*
 
 # Fortran module files
 *.mod
@@ -30,3 +44,50 @@
 *.exe
 *.out
 *.app
+
+# Protocol buffers
+*_pb2.py
+*.pb.h
+*.pb.cc
+*.pb
+*_grpc.py
+
+# Editor
+.vscode
+.idea/
+
+# Cquery
+.cquery_cached_index/
+compile_commands.json
+
+# Ctags and cscope
+tags
+TAGS
+CTAGS
+GTAGS
+GRTAGS
+GSYMS
+GPATH
+cscope.*
+
+# Python files
+*__pycache__*
+.pytest_cache
+
+# Mac files
+*.DS_Store
+
+# Test results
+test_temp_summary_event_file/
+*.dot
+*.dat
+*.svg
+*.perf
+*.info
+*.ckpt
+*.shp
+*.pkl
+.clangd
+mindspore_serving/version.py
+mindspore_serving/default_config.py
+mindspore_serving/.commit_id

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/mindspore"]
+	path = third_party/mindspore
+	url = https://gitee.com/mindspore/mindspore.git

CMakeLists.txt

@@ -0,0 +1,61 @@
+cmake_minimum_required(VERSION 3.14.1)
+project (MindSpore_Serving)
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.3.0)
+    message(FATAL_ERROR "GCC vesion ${CMAKE_CXX_COMPILER_VERSION} must not be less than 7.3.0")
+endif ()
+
+include(${CMAKE_SOURCE_DIR}/cmake/options.cmake)  # set compile options
+include(${CMAKE_SOURCE_DIR}/cmake/check_requirements.cmake) # check require party, like OpenSSL
+set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2")
+if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
+    add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0)
+endif ()
+
+if (ENABLE_PYTHON)
+    add_compile_definitions(ENABLE_PYTHON)
+endif()
+
+set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D_LIBCPP_DISABLE_EXTERN_TEMPLATE=1 -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -fPIC")
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+set(PYBIND11_CPP_STANDARD -std=c++17)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTION_CXX_FLAGS}")
+
+# compile third party: grpc, libevent, gtest, onnx
+include(${CMAKE_SOURCE_DIR}/cmake/mind_expression.cmake)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/securec/include)
+
+# find python3 packages
+include(${CMAKE_SOURCE_DIR}/cmake/dependency_utils.cmake)
+find_package(Python3 3.7 COMPONENTS Interpreter Development)
+if(Python3_FOUND)
+    set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}")
+    set(PYTHON_LIBRARIES "${Python3_LIBRARIES}")
+    if (WIN32)
+        if (Python3_DIR)
+            message("Python3_DIR set already: " ${Python3_DIR})
+        else()
+            string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN)
+            string(LENGTH "libpythonxx.a" Python3_NAME_LEN)
+            math(EXPR Python3_DIR_LEN  ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN})
+            string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR)
+            message("Python3_DIR: " ${Python3_DIR})
+        endif()
+        link_directories(${Python3_DIR})
+    endif()
+else()
+    find_python_package(py_inc py_lib)
+    set(PYTHON_INCLUDE_DIRS "${py_inc}")
+    set(PYTHON_LIBRARIES "${py_lib}")
+endif()
+message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}")
+message("PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}")
+include_directories(${PYTHON_INCLUDE_DIRS})
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
+find_package(Threads REQUIRED)
+add_subdirectory(mindspore_serving)
+include(cmake/package.cmake)

README.md

@@ -0,0 +1,150 @@
+# MindSpore-based Inference Service Deployment
+
+
+<!-- TOC -->
+
+- [MindSpore-based Inference Service Deployment](#mindspore-based-serving-service-deployment)
+    - [Overview](#overview)
+    - [Starting Serving](#starting-serving)
+    - [Application Example](#application-example)
+        - [Exporting Model](#exporting-model)
+        - [Starting Serving Inference](#starting-serving-serving)
+        - [Client Samples](#client-samples)
+            - [Python Client Sample](#python-client-sample)
+            - [C++ Client Sample](#cpp-client-sample)
+
+<!-- /TOC -->
+<a href="https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/advanced_use/serving.md" target="_blank"><img src="../_static/logo_source.png"></a>
+
+
+## Overview
+
+MindSpore Serving is a lightweight and high-performance service module that helps MindSpore developers efficiently deploy online serving services in the production environment. After completing model training using MindSpore, you can export the MindSpore model and use MindSpore Serving to create an serving service for the model. Currently, only Ascend 910 is supported.
+
+
+## Starting Serving
+After MindSpore is installed using `pip`, the Serving executable program is stored in `/{your python path}/lib/python3.7/site-packages/mindspore/ms_serving`.
+Run the following command to start Serving:
+```bash
+ms_serving [--help] [--model_path <MODEL_PATH>] [--model_name <MODEL_NAME>]
+                  [--port <PORT>] [--device_id <DEVICE_ID>]
+```
+Parameters are described as follows:
+
+|Parameter|Attribute|Function|Parameter Type|Default Value|Value Range|
+|---|---|---|---|---|---|
+|`--help`|Optional|Displays the help information about the startup command. |-|-|-|
+|`--model_path=<MODEL_PATH>`|Mandatory|Path for storing the model to be loaded. |String|Null|-|
+|`--model_name=<MODEL_NAME>`|Mandatory|Name of the model file to be loaded. |String|Null|-|
+|`--=port <PORT>`|Optional|Specifies the external Serving port number. |Integer|5500|1–65535|
+|`--device_id=<DEVICE_ID>`|Optional|Specifies device ID to be used.|Integer|0|0 to 7|
+
+ > Before running the startup command, add the path `/{your python path}/lib:/{your python path}/lib/python3.7/site-packages/mindspore/lib` to the environment variable `LD_LIBRARY_PATH`.
+
+## Application Example
+The following uses a simple network as an example to describe how to use MindSpore Serving.
+
+### Exporting Model
+Use [add_model.py](https://gitee.com/mindspore/mindspore/blob/master/serving/example/export_model/add_model.py) to build a network with only the Add operator and export the MindSpore serving deployment model.
+
+```python
+python add_model.py
+```
+Execute the script to generate the `tensor_add.mindir` file. The input of the model is two one-dimensional tensors with shape [2,2], and the output is the sum of the two input tensors.
+
+### Starting Serving Inference
+```bash
+ms_serving --model_path={model directory} --model_name=tensor_add.mindir
+```
+If the server prints the `MS Serving Listening on 0.0.0.0:5500` log, the Serving has loaded the serving model.
+
+### Client Samples
+#### <span name="python-client-sample">Python Client Sample</span>
+Obtain [ms_client.py](https://gitee.com/mindspore/mindspore/blob/master/serving/example/python_client/ms_client.py) and start the Python client.
+```bash
+python ms_client.py
+```
+
+If the following information is displayed, the Serving has correctly executed the serving of the Add network.
+```
+ms client received:
+[[2. 2.]
+ [2. 2.]]
+```
+
+#### <span name="cpp-client-sample">C++ Client Sample</span>
+1. Obtain an executable client sample program.
+
+    Download the [MindSpore source code](https://gitee.com/mindspore/mindspore). You can use either of the following methods to compile and obtain the client sample program:
+    + When MindSpore is compiled using the source code, the Serving C++ client sample program is generated. You can find the `ms_client` executable program in the `build/mindspore/serving/example/cpp_client` directory.
+    + Independent compilation
+
+        Preinstall [gRPC](https://gRPC.io).
+
+        Run the following command in the MindSpore source code path to compile a client sample program:
+        ```bash
+        cd mindspore/serving/example/cpp_client
+        mkdir build && cd build
+        cmake -D GRPC_PATH={grpc_install_dir} ..
+        make
+        ```
+        In the preceding command, `{grpc_install_dir}` indicates the gRPC installation path. Replace it with the actual gRPC installation path.
+
+2. Start the client.
+
+    Execute `ms_client` to send an serving request to the Serving.
+    ```bash
+    ./ms_client --target=localhost:5500
+    ```
+    If the following information is displayed, the Serving has correctly executed the serving of the Add network.
+    ```
+    Compute [[1, 2], [3, 4]] + [[1, 2], [3, 4]]
+    Add result is 2 4 6 8
+    client received: RPC OK
+    ```
+
+The client code consists of the following parts:
+
+1. Implement the client based on MSService::Stub and create a client instance.
+    ```
+    class MSClient {
+     public:
+      explicit MSClient(std::shared_ptr<Channel> channel) :  stub_(MSService::NewStub(channel)) {}
+     private:
+      std::unique_ptr<MSService::Stub> stub_;
+    };MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials()));
+    
+    MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials()));
+    
+    ```
+2. Build the request input parameter `Request`, output parameter `Reply`, and gRPC client `Context` based on the actual network input.
+    ```
+    PredictRequest request;
+    PredictReply reply;
+    ClientContext context;
+    
+    //construct tensor
+    Tensor data;
+    
+    //set shape
+    TensorShape shape;
+    shape.add_dims(4);
+    *data.mutable_tensor_shape() = shape;
+    
+    //set type
+    data.set_tensor_type(ms_serving::MS_FLOAT32);
+    std::vector<float> input_data{1, 2, 3, 4};
+    
+    //set datas
+    data.set_data(input_data.data(), input_data.size());
+    
+    //add tensor to request
+    *request.add_data() = data;
+    *request.add_data() = data;
+    ```
+3. Call the gRPC API to communicate with the Serving that has been started, and obtain the return value.
+    ```
+    Status status = stub_->Predict(&context, request, &reply);
+    ```
+
+For details about the complete code, see [ms_client](https://gitee.com/mindspore/mindspore/blob/master/serving/example/cpp_client/ms_client.cc). 

README_CN.md

@@ -0,0 +1,185 @@
+# MindSpore Serving
+
+[View English](./README.md)
+
+- [概述](#概述)
+- [安装部署](#安装部署)
+    - [安装MindSpore Serving](#安装MindSpore Serving)
+    - [配置环境变量](#配置环境变量)
+    - [部署MindSpore Serving](#部署MindSpore Serving)
+- [快速入门](#快速入门)
+    - [导出模型](#导出模型)
+    - [部署Serving推理服务](#部署serving推理服务)
+    - [执行推理](#执行推理)
+- [文档](#文档)
+    - [开发者教程](#开发者教程)
+- [社区](#社区)
+    - [治理](#治理)
+    - [交流](#交流)
+- [贡献](#贡献)
+- [版本说明](#版本说明)
+- [许可证](#许可证)
+
+## 概述
+
+MindSpore Serving是一个轻量级、高性能的服务模块，旨在帮助MindSpore开发者在生产环境中高效部署在线推理服务。当用户使用MindSpore完成模型训练后，导出MindSpore模型，即可使用MindSpore Serving创建该模型的推理服务。  
+
+**MindSpore Serving架构：**   
+当前MindSpore Serving服务节点分为client，master和worker。client为用户节点，下发推理服务命令。执行机worker部署了模型服务。当前仅支持Ascend 310和Ascend 910，后续会逐步支持GPU和CPU场景。master节点用来管理所有的执行机worker及其部署的模型信息，并进行任务管理与分发。master和worker可以部署在一个进程中，也可以部署在不同进程中。  
+<img src="docs/image/architecture.png" alt="MindSpore Architecture" width="600"/>   
+
+**MindSpore Serving提供以下功能：**
+- 支持客户端gRPC和RESTful接口
+- 支持组装模型的前处理和后处理
+- 支持batch功能
+- 提供客户端python简易接口
+
+## 安装部署
+MindSpore Serving依赖MindSpore训练推理框架，安装完[MindSpore](https://gitee.com/mindspore/mindspore#%E5%AE%89%E8%A3%85) ，再安装MindSpore Serving。
+
+### 安装MindSpore Serving
+使用pip命令安装，安装方式如下：
+
+**1、请从MindSpore Serving下载页面下载并安装whl包。**
+```python
+pip install mindspore_serving-1.0.0-cp37-cp37m-linux_x86_64.whl
+```
+**2、源码安装。**  
+下载[源码](https://gitee.com/mindspore/serving)。
+
+方式一，使用已安装或编译的MindSpore包：
+```shell
+sh build.sh -p $MINDSPORE_LIB_PATH
+```
+$MINDSPORE_LIB_PATH为mindspore软件包的安装路径下的lib路径，例：softwarepath/mindspore/lib，该路径包含mindspore运行依赖的库文件。
+
+方式二，编译Serving时编译配套的MindSpore包，需要配置MindSpore编译时的[环境变量](https://gitee.com/mindspore/docs/blob/master/install/mindspore_ascend_install_source.md#配置环境变量) ：
+```shell
+# ascend 310
+sh build.sh -eacl
+# ascend 910
+sh build.sh -ed
+```
+
+编译完后，在output/目录下找到安装包进行安装：
+```python
+pip install mindspore_serving-0.1.0-cp37-cp37m-linux_x86_64.whl
+```
+### 配置环境变量
+Asend 910环境上安装mindspore，需要完成[环境变量配置](https://gitee.com/mindspore/docs/blob/master/install/mindspore_ascend_install_pip.md#%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F)。   
+运行MindSpore Serving，还需要增加额外mindspore软件包的安装路径下的lib路径到LD_LIBRARY_PATH。
+```shell
+export LD_LIBRARY_PATH=$MINDSPORE_LIB_PATH:${LD_LIBRARY_PATH}
+```
+
+### 部署MindSpore Serving
+MindSpore Serving提供两种部署方式，用户可根据需要进行选择部署。
+
+**轻量级部署：**  
+服务端调用python接口直接启动推理进程（master和worker共进程），客户端直接连接推理服务后下发推理任务。  
+启动服务：
+```python
+import os
+from mindspore_serving import master
+from mindspore_serving import worker
+servable_dir = os.path.abspath(".")
+worker.start_servable_in_master(servable_dir, "xxx", device_id=0) 
+master.start_grpc_server("127.0.0.1", 5500)
+```
+
+**集群部署：**  
+服务端由master进程和worker进程组成，master用来管理集群内所有的worker节点，并进行推理任务的分发。  
+启动worker：
+```python
+import os
+from mindspore_serving import worker
+servable_dir = os.path.abspath(".")
+worker.start_servable(servable_dir, "lenet", device_id=0, 
+                       master_ip="127.0.0.1", master_port=5500,
+                       host_ip="127.0.0.1", host_port=5600)
+```
+启动master：
+```python
+from mindspore_serving import master
+master.start_grpc_server("127.0.0.1", 5500)
+```
+完成服务端部署后，即可启用客户端程序执行推理操作。
+
+## 快速入门
+以一个简单的Add网络为例，演示MindSpore Serving如何使用。
+
+### 导出模型
+使用[add_model.py](https://gitee.com/mindspore/serving/blob/master/mindspore_serving/example/add/export_model/add_model.py)，构造一个只有Add算子的网络，并导出MindSpore推理部署模型。
+
+```python
+python add_model.py
+```
+执行脚本，生成`tensor_add.mindir`文件，该模型的输入为两个shape为[2,2]的二维Tensor，输出结果是两个输入Tensor之和。
+
+### 部署Serving推理服务
+执行以下[python程序](https://gitee.com/mindspore/serving/blob/master/mindspore_serving/example/add/master_with_worker.py)，启动服务：
+```bash
+import os
+from mindspore_serving import master
+from mindspore_serving import worker
+def start():
+    servable_dir = os.path.abspath(".")
+    worker.start_servable_in_master(servable_dir, "add", device_id=0)
+    master.start_grpc_server("127.0.0.1", 5500)
+```
+启动过程需要使用servable_dir路径下的模型文件和配置文件，文件目录结果如下图所示：   
+<pre><font color="#268BD2"><b>add/</b></font>
+├── <font color="#268BD2"><b>1</b></font>
+│   └── tensor_add.mindir
+└── servable_config.py
+</pre>   
+其中，模型文件为上一步网络生成的，即`tensor_add.mindir`文件。配置文件为[servable_config.py](https://gitee.com/mindspore/serving/blob/master/mindspore_serving/example/add/add/servable_config.py)，其定义了模型的处理函数，包含前处理和后处理过程。
+当服务端打印日志`Serving gRPC start success, listening on 0.0.0.0:5500`时，表示Serving服务已加载推理模型完毕。
+
+### 执行推理
+使用[client.py](https://gitee.com/mindspore/serving/blob/master/mindspore_serving/example/add/client.py)，启动Python客户端。
+```bash
+python client.py
+```
+
+显示如下返回值说明Serving服务已正确执行Add网络的推理。
+```bash
+[{'y': array([[2. , 2.],
+        [2.,  2.]], dtype=float32)}]
+[{'y': array([[2. , 2.],
+        [2.,  2.]], dtype=float32)}]
+```
+
+## 文档
+
+### 开发者教程
+- [如何使用python接口开发客户端？](docs/GRPC.md)
+- [如何启动Restful服务进行推理？](docs/RESTful.md)
+- [如何实现模型前处理和后处理？](docs/MODEL.md)
+
+有关安装指南、教程和API的更多详细信息，请参阅[用户文档](https://gitee.com/mindspore/serving/docs)。
+
+## 社区
+
+### 治理
+
+查看MindSpore如何进行[开放治理](https://gitee.com/mindspore/community/blob/master/governance.md)。
+
+### 交流
+
+- [MindSpore Slack](https://join.slack.com/t/mindspore/shared_invite/zt-dgk65rli-3ex4xvS4wHX7UDmsQmfu8w) 开发者交流平台。
+- `#mindspore`IRC频道（仅用于会议记录）
+- 视频会议：待定
+- 邮件列表：<https://mailweb.mindspore.cn/postorius/lists>
+
+## 贡献
+
+欢迎参与贡献。
+
+## 版本说明
+
+版本说明请参阅[RELEASE](RELEASE.md)。
+
+## 许可证
+
+[Apache License 2.0](LICENSE)

RELEASE.md

@@ -0,0 +1,12 @@
+# Release 0.1.0-alpha
+
+## Main Features
+
+### Ascend 310 & Ascend 910 Serving Framework
+* Recommended OS: Ubuntu 16.04 (or later) or EulerOS 2.5 or EulerOS 2.8
+* Python version: 3.7.5
+* Provide gRPC and RESTful API
+* Support preprocessing & postprocessing
+* Support batching
+* Multiple instances
+* Version Control

build.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+set -e
+PROJECTPATH=$(cd "$(dirname $0)"; pwd)
+export BUILD_PATH="${PROJECTPATH}/build/"
+
+# print usage message
+usage()
+{
+  echo "Usage:"
+  echo "bash build.sh [-d] [-v] [-c] [-a on|off] [-j[n]] [-p]"
+  echo ""
+  echo "Options:"
+  echo "    -d Debug model"
+  echo "    -j[n] Set the threads when building (Default: -j8)"
+  echo "    -v Display build command. Run cpack with verbose output."
+  echo "    -c Enable code coverage, default off"
+  echo "    -a Enable ASAN, default off. Memory error detection tool."
+  echo "    -p MindSpore lib [mindspore_shared_lib] path."
+
+}
+
+# check value of input is 'on' or 'off'
+# usage: check_on_off arg_value arg_name
+check_on_off()
+{
+  if [[ "X$1" != "Xon" && "X$1" != "Xoff" ]]; then
+    echo "Invalid value $1 for option -$2"
+    usage
+    exit 1
+  fi
+}
+
+# check and set options
+checkopts()
+{
+  # Init default values of build options
+  THREAD_NUM=8
+  VERBOSE=""
+  DEBUG_MODE="off"
+  ENABLE_COVERAGE="off"
+  ENABLE_ASAN="off"
+  ENABLE_PYTHON="on"
+  MS_WHL_LIB_PATH=""
+  MS_BACKEND=""
+
+  # Process the options
+  while getopts 'dvcj:a:p:e:' opt
+  do
+    LOW_OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
+
+    case "${opt}" in
+      e)
+        echo "user opt: -e"${LOW_OPTARG}
+        if [[ "$OPTARG" != "" ]]; then
+          MS_BACKEND=$OPTARG
+        fi
+        ;;
+      p)
+        if [[ "$OPTARG"  != "" ]]; then
+          MS_WHL_LIB_PATH=$OPTARG
+        else
+          echo "Invalid value ${LOW_OPTARG} for option -e"
+          usage
+          exit 1
+        fi
+        ;;
+      d)
+        echo "user opt: -d"${LOW_OPTARG}
+        DEBUG_MODE="on"
+        ;;
+      j)
+        echo "user opt: -j"${LOW_OPTARG}
+        THREAD_NUM=$OPTARG
+        ;;
+      v)
+        echo "user opt: -v"${LOW_OPTARG}
+        VERBOSE="VERBOSE=1"
+        ;;
+      c)
+        check_on_off $OPTARG c
+        ENABLE_COVERAGE="$OPTARG"
+        ;;
+      a)
+        check_on_off $OPTARG a
+        ENABLE_ASAN="$OPTARG"
+        ;;
+      *)
+        echo "Unknown option ${opt}!"
+        usage
+        exit 1
+    esac
+  done
+}
+
+checkopts "$@"
+echo "---------------- MindSpore Serving: build start ----------------"
+mkdir -pv "${BUILD_PATH}/package/mindspore_serving/lib"
+if [[ "$MS_BACKEND" != "" ]]; then
+  git submodule update --init third_party/mindspore
+fi
+
+# Create building path
+build_mindspore_serving()
+{
+  echo "start build mindspore_serving project."
+  mkdir -pv "${BUILD_PATH}/mindspore_serving"
+  cd "${BUILD_PATH}/mindspore_serving"
+  CMAKE_ARGS="-DDEBUG_MODE=$DEBUG_MODE -DBUILD_PATH=$BUILD_PATH"
+  CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PYTHON=${ENABLE_PYTHON}"
+  CMAKE_ARGS="${CMAKE_ARGS} -DTHREAD_NUM=${THREAD_NUM}"
+  if [[ "X$ENABLE_COVERAGE" = "Xon" ]]; then
+    CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_COVERAGE=ON"
+  fi
+  if [[ "X$ENABLE_ASAN" = "Xon" ]]; then
+      CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_ASAN=ON"
+  fi
+  if [[ "$MS_BACKEND" != "" ]]; then
+    CMAKE_ARGS="${CMAKE_ARGS} -DMS_BACKEND=${MS_BACKEND}"
+  fi
+  if [[ "$MS_WHL_LIB_PATH" != "" ]]; then
+    CMAKE_ARGS="${CMAKE_ARGS} -DMS_WHL_LIB_PATH=${MS_WHL_LIB_PATH}"
+  fi
+  echo "${CMAKE_ARGS}"
+  cmake ${CMAKE_ARGS} ../..
+  if [[ -n "$VERBOSE" ]]; then
+    CMAKE_VERBOSE="--verbose"
+  fi
+  cmake --build . --target package ${CMAKE_VERBOSE} -j$THREAD_NUM
+  echo "success building mindspore_serving project!"
+}
+
+build_mindspore_serving
+
+echo "---------------- mindspore_serving: build end   ----------------"

cmake/check_requirements.cmake

@@ -0,0 +1,39 @@
+## define customized find fucntions, print customized error messages
+function(find_required_package pkg_name)
+    find_package(${pkg_name})
+    if (NOT ${pkg_name}_FOUND)
+        message(FATAL_ERROR "Required package ${pkg_name} not found, please install the package and try building mindspore_serving again.")
+    endif()
+endfunction()
+
+## find python, quit if the found python is static
+set(Python3_USE_STATIC_LIBS FALSE)
+find_package(Python3 COMPONENTS Interpreter Development)
+if (Python3_FOUND)
+    message("Python3 found, version: ${Python3_VERSION}")
+    message("Python3 library path: ${Python3_LIBRARY_DIRS}")
+    message("Python3 interpreter: ${Python3_EXECUTABLE}")
+    else()
+    message(FATAL_ERROR "Python3 not found, please install Python>=3.7.5, and set --enable-shared "
+            "if you are building Python locally")
+endif ()
+
+## packages used both on windows and linux
+if (DEFINED ENV{MS_PATCH_PATH})
+    find_program(Patch_EXECUTABLE patch PATHS $ENV{MS_PATCH_PATH})
+    set(Patch_FOUND ${Patch_EXECUTABLE})
+else ()
+    find_package(Patch)
+endif ()
+if (NOT Patch_FOUND)
+    message(FATAL_ERROR "Patch not found, please set environment variable MS_PATCH_PATH to path where Patch is located, "
+            "usually found in GIT_PATH/usr/bin on Windows")
+endif ()
+message(PATCH_EXECUTABLE = ${Patch_EXECUTABLE})
+
+find_required_package(Threads)
+
+## packages used on Linux
+if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
+    find_required_package(OpenSSL)
+endif()

cmake/dependency_ms.cmake

@@ -0,0 +1,13 @@
+# Compile MindSpore
+
+message(STATUS "**********begin to compile MindSpore**********")
+set(MS_SOURCE_DIR ${CMAKE_SOURCE_DIR}/third_party/mindspore)
+message(STATUS "MindSpore dir: ${MS_SOURCE_DIR}")
+message(STATUS "MindSpore compile method: -e${MS_BACKEND}")
+message(STATUS "MindSpore compile thread num: -j${THREAD_NUM}")
+
+execute_process(
+        COMMAND bash ${MS_SOURCE_DIR}/build.sh -e${MS_BACKEND} -j${THREAD_NUM}
+        WORKING_DIRECTORY ${MS_SOURCE_DIR}
+)
+message(STATUS "**********end to compile MindSpore**********")

cmake/dependency_securec.cmake

@@ -0,0 +1,22 @@
+# securec library
+#
+#
+# SECUREC_LIBRARY
+#
+
+if (NOT TARGET securec)
+  set(_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE ${CMAKE_POSITION_INDEPENDENT_CODE})
+  set(_ms_tmp_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+
+  set(CMAKE_C_FLAGS "${SECURE_CXX_FLAGS}")
+  if (CMAKE_SYSTEM_NAME MATCHES "Windows")
+    add_compile_definitions(SECUREC_ONLY_DECLARE_MEMSET)
+  endif()
+  add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/securec ${CMAKE_BINARY_DIR}/securec)
+  set(CMAKE_POSITION_INDEPENDENT_CODE ${_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE})
+  set(CMAKE_C_FLAGS ${_ms_tmp_CMAKE_C_FLAGS})
+endif()
+
+include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/securec/include)
+
+set(SECUREC_LIBRARY securec)

cmake/dependency_utils.cmake

@@ -0,0 +1,25 @@
+# MS Utils
+#
+
+function(find_python_package out_inc out_lib)
+  # Use PYTHON_EXECUTABLE if it is defined, otherwise default to python
+  if ("${PYTHON_EXECUTABLE}" STREQUAL "")
+    set(PYTHON_EXECUTABLE "python3")
+  else()
+    set(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}")
+  endif()
+
+  execute_process(
+          COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())"
+          RESULT_VARIABLE result
+          OUTPUT_VARIABLE inc)
+  string(STRIP "${inc}" inc)
+  set(${out_inc} ${inc} PARENT_SCOPE)
+  
+  execute_process(
+          COMMAND "${PYTHON_EXECUTABLE}" -c "import distutils.sysconfig as sysconfig; import os; print(os.path.join(sysconfig.get_config_var('LIBDIR'), sysconfig.get_config_var('LDLIBRARY')))"
+          RESULT_VARIABLE result
+          OUTPUT_VARIABLE lib)
+  string(STRIP "${lib}" lib)
+  set(${out_lib} ${lib} PARENT_SCOPE)
+endfunction()

cmake/external_libs/absl.cmake

@@ -0,0 +1,14 @@
+mindspore_add_pkg(absl
+        VER 20200225.2
+        LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access
+        URL https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz
+        MD5 73f2b6e72f1599a9139170c29482ddc4
+        CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE)
+
+include_directories(${absl_INC})
+
+add_library(mindspore_serving::absl_strings ALIAS absl::absl_strings)
+add_library(mindspore_serving::absl_throw_delegate ALIAS absl::absl_throw_delegate)
+add_library(mindspore_serving::absl_raw_logging_internal ALIAS absl::absl_raw_logging_internal)
+add_library(mindspore_serving::absl_int128 ALIAS absl::absl_int128)
+add_library(mindspore_serving::absl_bad_optional_access ALIAS absl::absl_bad_optional_access)

cmake/external_libs/c-ares.cmake

@@ -0,0 +1,12 @@
+mindspore_add_pkg(c-ares
+        VER 1.15.0
+        LIBS cares 
+        URL https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz
+        MD5 d2391da274653f7643270623e822dff7
+        CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release
+        -DCARES_SHARED:BOOL=OFF
+        -DCARES_STATIC:BOOL=ON
+        -DCARES_STATIC_PIC:BOOL=ON)
+
+include_directories(${c-ares_INC})
+add_library(mindspore_serving::cares ALIAS c-ares::cares)

cmake/external_libs/glog.cmake

@@ -0,0 +1,17 @@
+set(glog_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 ${SECURE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+set(glog_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
+if (ENABLE_GITEE)
+    set(REQ_URL "https://gitee.com/mirrors/glog/repository/archive/v0.4.0.tar.gz")
+    set(MD5 "22fe340ddc231e6c8e46bc295320f8ee")
+else()
+    set(REQ_URL "https://github.com/google/glog/archive/v0.4.0.tar.gz")
+    set(MD5 "0daea8785e6df922d7887755c3d100d0")
+endif ()
+mindspore_add_pkg(glog
+        VER 0.4.0
+        LIBS glog
+        URL ${REQ_URL}
+        MD5 ${MD5}
+        CMAKE_OPTION -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON -DWITH_GFLAGS=OFF)
+include_directories(${glog_INC})
+add_library(mindspore::glog ALIAS glog::glog)

cmake/external_libs/grpc.cmake

@@ -0,0 +1,110 @@
+set(grpc_USE_STATIC_LIBS ON)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+else()
+    set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
+endif()
+
+set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
+
+
+if (EXISTS ${protobuf_ROOT}/lib64)
+  set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf")
+else()
+  set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf")
+endif()
+message("grpc using Protobuf_DIR : " ${_FINDPACKAGE_PROTOBUF_CONFIG_DIR})
+
+if (EXISTS ${absl_ROOT}/lib64)
+  set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl")
+else()
+  set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl")
+endif()
+message("grpc using absl_DIR : " ${_FINDPACKAGE_ABSL_CONFIG_DIR})
+
+set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "")
+if (OPENSSL_ROOT_DIR)
+  set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}")
+endif()
+
+mindspore_add_pkg(grpc
+        VER 1.27.3
+        LIBS grpc++ grpc gpr upb address_sorting
+        EXE grpc_cpp_plugin
+        URL https://github.com/grpc/grpc/archive/v1.27.3.tar.gz
+        MD5 0c6c3fc8682d4262dd0e5e6fabe1a7e2
+        CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release
+        -DgRPC_INSTALL:BOOL=ON
+        -DgRPC_BUILD_TESTS:BOOL=OFF
+        -DgRPC_PROTOBUF_PROVIDER:STRING=package
+        -DgRPC_PROTOBUF_PACKAGE_TYPE:STRING=CONFIG
+        -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR}
+        -DgRPC_ZLIB_PROVIDER:STRING=package
+        -DZLIB_ROOT:PATH=${zlib_ROOT}
+        -DgRPC_ABSL_PROVIDER:STRING=package
+        -Dabsl_DIR:PATH=${_FINDPACKAGE_ABSL_CONFIG_DIR}
+        -DgRPC_CARES_PROVIDER:STRING=package
+        -Dc-ares_DIR:PATH=${c-ares_ROOT}/lib/cmake/c-ares
+        -DgRPC_SSL_PROVIDER:STRING=package
+        ${_CMAKE_ARGS_OPENSSL_ROOT_DIR}
+        )
+
+include_directories(${grpc_INC})
+
+add_library(mindspore_serving::grpc++ ALIAS grpc::grpc++)
+
+# link other grpc libs
+target_link_libraries(grpc::grpc++ INTERFACE grpc::grpc grpc::gpr grpc::upb grpc::address_sorting)
+
+# link built dependencies
+target_link_libraries(grpc::grpc++ INTERFACE mindspore_serving::z)
+target_link_libraries(grpc::grpc++ INTERFACE mindspore_serving::cares)
+target_link_libraries(grpc::grpc++ INTERFACE mindspore_serving::absl_strings mindspore_serving::absl_throw_delegate
+                      mindspore_serving::absl_raw_logging_internal mindspore_serving::absl_int128 mindspore_serving::absl_bad_optional_access)
+
+# link system openssl
+find_package(OpenSSL REQUIRED)
+target_link_libraries(grpc::grpc++ INTERFACE OpenSSL::SSL OpenSSL::Crypto)
+
+
+function(ms_grpc_generate c_var h_var)
+    if(NOT ARGN)
+        message(SEND_ERROR "Error: ms_grpc_generate() called without any proto files")
+        return()
+    endif()
+
+    set(${c_var})
+    set(${h_var})
+
+    foreach(file ${ARGN})
+        get_filename_component(abs_file ${file} ABSOLUTE)
+        get_filename_component(file_name ${file} NAME_WE)
+        get_filename_component(file_dir ${abs_file} PATH)
+        file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir})
+
+        list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc")
+        list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h")
+        list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.cc")
+        list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.h")
+
+        add_custom_command(
+                OUTPUT "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc"
+                "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h"
+                "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.cc"
+                "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.h"
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto"
+                COMMAND protobuf::protoc --version
+                COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto
+                --grpc_out=${CMAKE_BINARY_DIR}/proto --plugin=protoc-gen-grpc=$<TARGET_FILE:grpc::grpc_cpp_plugin> ${abs_file}
+                DEPENDS protobuf::protoc grpc::grpc_cpp_plugin ${abs_file}
+                COMMENT "Running C++ gRPC compiler on ${file}" VERBATIM)
+    endforeach()
+
+    set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
+    set(${c_var} ${${c_var}} PARENT_SCOPE)
+    set(${h_var} ${${h_var}} PARENT_SCOPE)
+
+endfunction()

cmake/external_libs/gtest.cmake

@@ -0,0 +1,13 @@
+set(gtest_CXXFLAGS "-D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
+set(gtest_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
+mindspore_add_pkg(gtest
+        VER 1.8.0
+        LIBS gtest
+        URL https://github.com/google/googletest/archive/release-1.8.0.tar.gz
+        MD5 16877098823401d1bf2ed7891d7dce36
+        CMAKE_OPTION -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON
+        -DCMAKE_MACOSX_RPATH=TRUE -Dgtest_disable_pthreads=ON)
+include_directories(${gtest_INC})
+add_library(mindspore_serving::gtest ALIAS gtest::gtest)
+file(COPY ${gtest_LIBPATH}/libgtest${CMAKE_SHARED_LIBRARY_SUFFIX} DESTINATION ${CMAKE_BINARY_DIR}/googletest/googlemock/gtest)
+file(COPY ${gtest_LIBPATH}/libgtest_main${CMAKE_SHARED_LIBRARY_SUFFIX} DESTINATION ${CMAKE_BINARY_DIR}/googletest/googlemock/gtest)

cmake/external_libs/json.cmake

@@ -0,0 +1,9 @@
+set(nlohmann_json_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
+set(nlohmann_json_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
+mindspore_add_pkg(nlohmann_json
+        VER 3.6.1
+        HEAD_ONLY ./
+        URL https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip
+        MD5 0dc903888211db3a0f170304cd9f3a89)
+include_directories(${nlohmann_json_INC})
+add_library(mindspore_serving::json ALIAS nlohmann_json)

cmake/external_libs/libevent.cmake

@@ -0,0 +1,13 @@
+set(libevent_CFLAGS "-fstack-protector-all -D_FORTIFY_SOURCE=2 -O2")
+set(libevent_LDFLAGS "-Wl,-z,now")
+mindspore_add_pkg(libevent
+        VER 2.1.12
+        LIBS event event_pthreads
+        URL https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
+        MD5 b5333f021f880fe76490d8a799cd79f4
+        CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_TESTING=OFF)
+
+include_directories(${libevent_INC}) # 将指定目录添加到编译器的头文件搜索路径之下，指定的目录被解释成当前源码路径的相对路径。
+
+add_library(mindspore_serving::event ALIAS libevent::event)
+add_library(mindspore_serving::event_pthreads ALIAS libevent::event_pthreads)

cmake/external_libs/protobuf.cmake

@@ -0,0 +1,120 @@
+set(protobuf_USE_STATIC_LIBS ON)
+if (BUILD_LITE)
+    set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+else(BUILD_LITE)
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+        set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+    elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+        set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2")
+    else()
+        set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
+    endif()
+endif(BUILD_LITE)
+
+set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
+set(_ms_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS})
+string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+
+mindspore_add_pkg(protobuf
+        VER 3.8.0
+        LIBS protobuf
+        EXE protoc
+        URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
+        MD5 3d9e32700639618a4d2d342c99d4507a
+        CMAKE_PATH cmake/
+        CMAKE_OPTION -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF)
+
+include_directories(${protobuf_INC})
+add_library(mindspore_serving::protobuf ALIAS protobuf::protobuf)
+set(CMAKE_CXX_FLAGS  ${_ms_tmp_CMAKE_CXX_FLAGS})
+
+function(ms_protobuf_generate c_var h_var)
+    if(NOT ARGN)
+        message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files")
+        return()
+    endif()
+
+    set(${c_var})
+    set(${h_var})
+
+    foreach(file ${ARGN})
+        get_filename_component(abs_file ${file} ABSOLUTE)
+        get_filename_component(file_name ${file} NAME_WE)
+        get_filename_component(file_dir ${abs_file} PATH)
+        file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir})
+
+        list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc")
+        list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h")
+
+        add_custom_command(
+                OUTPUT "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc"
+                "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h"
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto"
+                COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                DEPENDS protobuf::protoc ${abs_file}
+                COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM)
+    endforeach()
+
+    set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
+    set(${c_var} ${${c_var}} PARENT_SCOPE)
+    set(${h_var} ${${h_var}} PARENT_SCOPE)
+
+endfunction()
+
+function(ms_protobuf_generate_py c_var h_var py_var)
+    if(NOT ARGN)
+        message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files")
+        return()
+    endif()
+
+    set(${c_var})
+    set(${h_var})
+    set(${py_var})
+
+    foreach(file ${ARGN})
+        get_filename_component(abs_file ${file} ABSOLUTE)
+        get_filename_component(file_name ${file} NAME_WE)
+        get_filename_component(file_dir ${abs_file} PATH)
+
+        list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc")
+        list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h")
+        list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py")
+        if (WIN32)
+            add_custom_command(
+                    OUTPUT "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc"
+                    "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h"
+                    "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py"
+                    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                    COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto"
+                    COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                    COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                    COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                    COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/"  "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py"
+                    COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore_serving/train/"
+                    DEPENDS protobuf::protoc ${abs_file}
+                    COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
+        else()
+            add_custom_command(
+                    OUTPUT "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc"
+                    "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h"
+                    "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py"
+                    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                    COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto"
+                    COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                    COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                    COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/proto ${abs_file}
+                    COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/"  "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py"
+                    COMMAND cp "${CMAKE_BINARY_DIR}/proto/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore_serving/train/"
+                    DEPENDS protobuf::protoc ${abs_file}
+                    COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM)
+        endif()
+    endforeach()
+    set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE)
+    set(${c_var} ${${c_var}} PARENT_SCOPE)
+    set(${h_var} ${${h_var}} PARENT_SCOPE)
+    set(${py_var} ${${py_var}} PARENT_SCOPE)
+
+endfunction()

cmake/external_libs/pybind11.cmake

@@ -0,0 +1,12 @@
+set(pybind11_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
+set(pybind11_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
+mindspore_add_pkg(pybind11
+        VER 2.4.3
+        URL https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz
+        MD5 62254c40f89925bb894be421fe4cdef2
+        CMAKE_OPTION -DPYBIND11_TEST=OFF -DPYBIND11_LTO_CXX_FLAGS=FALSE
+        )
+include_directories(${pybind11_INC})
+find_package(pybind11 REQUIRED)
+set_property(TARGET pybind11::module PROPERTY IMPORTED_GLOBAL TRUE)
+add_library(mindspore_serving::pybind11_module ALIAS pybind11::module)

cmake/external_libs/zlib.cmake

@@ -0,0 +1,9 @@
+mindspore_add_pkg(zlib
+        VER 1.2.11
+        LIBS z
+        URL https://github.com/madler/zlib/archive/v1.2.11.tar.gz
+        MD5 0095d2d2d1f3442ce1318336637b695f
+        CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release)
+
+include_directories(${zlib_INC})
+add_library(mindspore_serving::z ALIAS zlib::z)

cmake/mind_expression.cmake

@@ -0,0 +1,32 @@
+set(SECURE_CXX_FLAGS "")
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+	set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
+endif()
+set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS})
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
+
+# define third party library download function
+include(cmake/utils.cmake)
+
+include(${CMAKE_SOURCE_DIR}/cmake/dependency_securec.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/json.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake)
+
+# build dependencies of gRPC
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/absl.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/c-ares.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zlib.cmake)
+# build gRPC
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/grpc.cmake)
+# build event
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/libevent.cmake)
+
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pybind11.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/gtest.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake)
+
+set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS_F})
+
+if(MS_BACKEND)
+	include(${CMAKE_SOURCE_DIR}/cmake/dependency_ms.cmake)
+endif()

cmake/options.cmake

@@ -0,0 +1,53 @@
+option(DEBUG_MODE "Debug mode, default off" OFF)
+option(ENABLE_COVERAGE "Enable code coverage report" OFF)
+option(ENABLE_PYTHON "Enable python" ON)
+option(ENABLE_ASAN "Enable Google Sanitizer to find memory bugs")
+option(MS_WHL_LIB_PATH "MindSpore lib path")
+option(MS_BACKEND "Compile MindSpore")
+
+if(MS_WHL_LIB_PATH)
+    message("MindSpore whl lib path: " ${MS_WHL_LIB_PATH})
+elseif(MS_BACKEND)
+    message("MindSpore backend method: " ${MS_BACKEND})
+else()
+    message( FATAL_ERROR "Please confirm how to use MindSpore.")
+endif()
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND Linux)
+    set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
+endif()
+
+if (ENABLE_COVERAGE)
+    set(COVERAGE_COMPILER_FLAGS "-g --coverage -fprofile-arcs -ftest-coverage")
+    set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}")
+endif()
+
+if (ENABLE_ASAN)
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+        set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fsanitize=address -fsanitize-recover=address -fno-omit-frame-pointer -fsanitize=undefined")
+    else()
+        set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -static-libsan -fsanitize=undefined")
+    endif()
+endif()
+
+if (DEBUG_MODE)
+    set(CMAKE_BUILD_TYPE "Debug")
+    add_compile_definitions(MEM_REUSE_DEBUG)
+else()
+    set(CMAKE_BUILD_TYPE "Release")
+endif()
+
+if ((CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") OR (CMAKE_BUILD_TYPE STREQUAL Release))
+    set(PYBIND11_LTO_CXX_FLAGS FALSE)
+endif()
+
+if (NOT BUILD_PATH)
+    set(BUILD_PATH "${CMAKE_SOURCE_DIR}/build")
+endif()
+
+if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
+    set(MS_BUILD_GRPC ON)
+endif()
+
+add_compile_definitions(USE_GLOG)
+

cmake/package.cmake

@@ -0,0 +1,70 @@
+# include dependency
+include(CMakePackageConfigHelpers)
+include(GNUInstallDirs)
+
+# set package information
+set(CPACK_PACKAGE_NAME ${PROJECT_NAME})
+set(CPACK_GENERATOR "External")
+set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/package_script.cmake)
+set(CPACK_EXTERNAL_ENABLE_STAGING true)
+set(CPACK_TEMPORARY_PACKAGE_FILE_NAME ${CMAKE_SOURCE_DIR}/build/package/mindspore_serving)
+set(CPACK_TEMPORARY_INSTALL_DIRECTORY ${CMAKE_SOURCE_DIR}/build/package/mindspore_serving)
+
+set(CPACK_MS_PACKAGE_NAME "mindspore_serving")
+include(CPack)
+
+# set install path
+set(INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Installation directory for libraries")
+set(INSTALL_PY_DIR ".")
+set(INSTALL_BASE_DIR ".")
+set(INSTALL_LIB_DIR "lib")
+
+file(GLOB_RECURSE LIBEVENT_LIB_LIST
+        ${libevent_LIBPATH}/libevent*
+        ${libevent_LIBPATH}/libevent_pthreads*
+        )
+install(
+        FILES ${LIBEVENT_LIB_LIST}
+        DESTINATION ${INSTALL_LIB_DIR}
+        COMPONENT mindspore_serving
+)
+
+file(GLOB_RECURSE GLOG_LIB_LIST ${glog_LIBPATH}/libglog*)
+install(
+        FILES ${GLOG_LIB_LIST}
+        DESTINATION ${INSTALL_LIB_DIR}
+        COMPONENT mindspore_serving
+)
+
+# set python files
+file(GLOB MS_PY_LIST ${CMAKE_SOURCE_DIR}/mindspore_serving/*.py)
+install(
+    FILES ${MS_PY_LIST}
+    DESTINATION ${INSTALL_PY_DIR}
+    COMPONENT mindspore_serving
+)
+
+install(
+    TARGETS _mindspore_serving
+    DESTINATION ${INSTALL_BASE_DIR}
+    COMPONENT mindspore_serving
+)
+install(
+    TARGETS serving_common
+    DESTINATION ${INSTALL_LIB_DIR}
+    COMPONENT mindspore_serving
+)
+install(
+    DIRECTORY
+    ${CMAKE_SOURCE_DIR}/mindspore_serving/master
+    ${CMAKE_SOURCE_DIR}/mindspore_serving/worker
+    ${CMAKE_SOURCE_DIR}/mindspore_serving/client
+    DESTINATION ${INSTALL_PY_DIR}
+    COMPONENT mindspore_serving
+)
+install(
+    FILES ${CMAKE_SOURCE_DIR}/build/mindspore_serving/mindspore_serving/mindspore_serving/proto/ms_service_pb2.py
+          ${CMAKE_SOURCE_DIR}/build/mindspore_serving/mindspore_serving/mindspore_serving/proto/ms_service_pb2_grpc.py
+    DESTINATION ${INSTALL_PY_DIR}/proto
+    COMPONENT mindspore_serving
+)

cmake/package_script.cmake

@@ -0,0 +1,75 @@
+# find exec
+find_package(Python3 3.7 COMPONENTS Interpreter)
+if (NOT Python3_FOUND)
+    message(FATAL_ERROR "No python3 found.")
+endif ()
+
+set(PYTHON ${Python3_EXECUTABLE})
+set(PYTHON_VERSION ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR})
+
+if (NOT PYTHON_VERSION MATCHES "3.7")
+    message(FATAL_ERROR "FIND PYTHON VERSION ${PYTHON_VERSION} BUT CAN NOT MATCH PYTHON VERSION 3.7")
+endif ()
+
+find_package(Git)
+if (NOT GIT_FOUND)
+    message("No git found.")
+    return ()
+endif ()
+set(GIT ${GIT_EXECUTABLE})
+
+# set path
+set(MS_ROOT_DIR ${CPACK_PACKAGE_DIRECTORY}/../../)
+set(MS_PACK_ROOT_DIR ${MS_ROOT_DIR}/build/package)
+# set package file name
+if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+    if (PYTHON_VERSION MATCHES "3.7")
+        set(PY_TAGS "cp37-cp37m")
+    else ()
+        message("Could not find 'Python 3.7'")
+        return()
+    endif ()
+    string(TOLOWER linux_${CMAKE_HOST_SYSTEM_PROCESSOR} PLATFORM_TAG)
+else ()
+    message(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}")
+endif ()
+
+# get git commit id
+set(GIT_COMMIT_ID "")
+execute_process(
+    COMMAND ${GIT} log --format='[sha1]:%h,[branch]:%d' --abbrev=8 -1
+    OUTPUT_VARIABLE GIT_COMMIT_ID
+    WORKING_DIRECTORY ${MS_ROOT_DIR}
+    ERROR_QUIET)
+string(REPLACE " " "" GIT_COMMIT_ID ${GIT_COMMIT_ID})
+
+set(ENV{MS_PACKAGE_NAME} ${CPACK_MS_PACKAGE_NAME})
+set(ENV{COMMIT_ID} ${GIT_COMMIT_ID})
+
+execute_process(
+    COMMAND ${PYTHON} ${MS_ROOT_DIR}/setup.py "bdist_wheel"
+    WORKING_DIRECTORY ${MS_PACK_ROOT_DIR}
+)
+
+# finally
+set(PACKAGE_NAME ${CPACK_MS_PACKAGE_NAME})
+if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
+    string(REPLACE "-" "_" PACKAGE_NAME ${PACKAGE_NAME})
+    execute_process(
+        COMMAND chmod -R 700 ${MS_PACK_ROOT_DIR}/mindspore_serving/
+        COMMAND chmod -R 700 ${MS_PACK_ROOT_DIR}/${PACKAGE_NAME}.egg-info/
+    )
+endif ()
+
+file(GLOB WHL_FILE ${MS_PACK_ROOT_DIR}/dist/*.whl)
+get_filename_component(ORIGIN_FILE_NAME ${WHL_FILE} NAME)
+string(REPLACE "-" ";" ORIGIN_FILE_NAME ${ORIGIN_FILE_NAME})
+list(GET ORIGIN_FILE_NAME 1 VERSION)
+set(NEW_FILE_NAME ${PACKAGE_NAME}-${VERSION}-${PY_TAGS}-${PLATFORM_TAG}.whl)
+file(RENAME ${WHL_FILE} ${MS_PACK_ROOT_DIR}/${NEW_FILE_NAME})
+file(REMOVE_RECURSE ${MS_ROOT_DIR}/output)
+file(MAKE_DIRECTORY ${MS_ROOT_DIR}/output)
+file(COPY ${MS_PACK_ROOT_DIR}/${NEW_FILE_NAME} DESTINATION ${MS_ROOT_DIR}/output/)
+
+file(SHA256 ${MS_ROOT_DIR}/output/${NEW_FILE_NAME} SHA256_VAR)
+file(WRITE ${MS_ROOT_DIR}/output/${NEW_FILE_NAME}.sha256 ${SHA256_VAR} " " ${NEW_FILE_NAME})

cmake/utils.cmake

@@ -0,0 +1,422 @@
+include(FetchContent) # 下载第三方库
+set(FETCHCONTENT_QUIET OFF)
+
+function(mindspore_add_submodule_obj des_submodule_objs sub_dir submodule_name_obj)
+
+    add_subdirectory(${sub_dir})
+
+    if(NOT TARGET ${submodule_name_obj})
+        message(FATAL_ERROR "Can not find submodule '${submodule_name_obj}'. in ${CMAKE_CURRENT_LIST_FILE}")
+    endif()
+    if("$<TARGET_OBJECTS:${submodule_name_obj}>" IN_LIST ${des_submodule_objs})
+        message(FATAL_ERROR "submodule '${submodule_name_obj}' added more than once. in ${CMAKE_CURRENT_LIST_FILE}")
+    endif()
+
+    set(${des_submodule_objs} ${${des_submodule_objs}} $<TARGET_OBJECTS:${submodule_name_obj}> PARENT_SCOPE)
+
+endfunction()
+
+if (DEFINED ENV{MSLIBS_CACHE_PATH})
+    set(_MS_LIB_CACHE  $ENV{MSLIBS_CACHE_PATH})
+else()
+    set(_MS_LIB_CACHE ${CMAKE_BINARY_DIR}/.mslib)
+endif ()
+message("MS LIBS CACHE PATH:  ${_MS_LIB_CACHE}")
+
+if (NOT EXISTS ${_MS_LIB_CACHE})
+    file(MAKE_DIRECTORY ${_MS_LIB_CACHE})
+endif ()
+
+if (DEFINED ENV{MSLIBS_SERVER})  # export MSLIBS_SERVER=49.4.0.74
+    set(LOCAL_LIBS_SERVER  $ENV{MSLIBS_SERVER})
+    message("LOCAL_LIBS_SERVER:  ${LOCAL_LIBS_SERVER}")
+endif ()
+
+include(ProcessorCount) # 确定处理器/核的数量并将值保存在${var}中
+ProcessorCount(N)
+if (JOBS)
+    set(THNUM ${JOBS})
+else()
+    set(JOBS 8)
+    if (${JOBS} GREATER ${N})
+        set(THNUM ${N})
+    else()
+        set(THNUM ${JOBS})
+    endif()
+endif ()
+message("set make thread num: ${THNUM}")
+
+if(LOCAL_LIBS_SERVER)
+    if (NOT ENV{no_proxy})
+        set(ENV{no_proxy} "${LOCAL_LIBS_SERVER}")
+    else()
+        string(FIND $ENV{no_proxy} ${LOCAL_LIBS_SERVER} IP_POS)
+        if (${IP_POS} EQUAL -1)
+            set(ENV{no_proxy} "$ENV{no_proxy},${LOCAL_LIBS_SERVER}")
+        endif ()
+    endif ()
+endif()
+
+function(__download_pkg pkg_name pkg_url pkg_md5)
+
+    if(LOCAL_LIBS_SERVER)
+        get_filename_component(_URL_FILE_NAME ${pkg_url} NAME)
+        set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${_URL_FILE_NAME}" ${pkg_url})
+    endif()
+
+    FetchContent_Declare(  # 获取项目。可以是一个URL也可以是一个Git仓库。
+            ${pkg_name}
+            URL      ${pkg_url}
+            URL_HASH MD5=${pkg_md5}
+    )
+    FetchContent_GetProperties(${pkg_name}) # 获取我们需要的变量MyName_*。
+    message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}")
+    if(NOT ${pkg_name}_POPULATED)
+        FetchContent_Populate(${pkg_name}) # 将信息记录在可以随时查询的全局属性中
+        set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE)
+    endif()
+
+endfunction()
+
+function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
+
+    if(LOCAL_LIBS_SERVER)
+        set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${pkg_git_commit}")
+        FetchContent_Declare(
+                ${pkg_name}
+                URL      ${pkg_url}
+                URL_HASH MD5=${pkg_md5}
+    )
+    else()
+	FetchContent_Declare(
+            ${pkg_name}
+	    GIT_REPOSITORY      ${pkg_url}
+	    GIT_TAG             ${pkg_git_commit})
+    endif()
+    FetchContent_GetProperties(${pkg_name})
+    message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}")
+    if(NOT ${pkg_name}_POPULATED)
+        FetchContent_Populate(${pkg_name})
+        set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE)
+    endif()
+
+endfunction()
+
+
+function(__find_pkg_then_add_target pkg_name pkg_exe lib_path)
+
+    unset(${pkg_name}_LIBS)
+
+    message("_FIND:${${pkg_name}_BASE_DIR}")
+
+    if(pkg_exe)
+        # find_program：该命令用于查找程序。<VAR>创建名为的缓存条目以存储此命令的结果。
+        # 如果找到程序，则结果存储在变量中，除非清除变量，否则将不会重复搜索。如果什么也没找到，结果将是<VAR>-NOTFOUND。
+        find_program(${pkg_exe}_EXE ${pkg_exe} PATHS ${${pkg_name}_BASE_DIR}/bin NO_DEFAULT_PATH)
+        if(NOT ${pkg_exe}_EXE)
+            return()
+        endif()
+        # add_executable： 使用给定的源文件，为工程引入一个可执行文件。
+        # IMPORTED：一个导入的可执行目标引用了一个位于工程之外的可执行文件。
+        add_executable(${pkg_name}::${pkg_exe} IMPORTED GLOBAL)
+        set_target_properties(${pkg_name}::${pkg_exe} PROPERTIES
+                IMPORTED_LOCATION ${${pkg_exe}_EXE}
+                )
+        message("found ${${pkg_exe}_EXE}")
+    endif()
+
+    foreach(_LIB_NAME ${ARGN})
+        set(_LIB_SEARCH_NAME ${_LIB_NAME})
+        set(_LIB_TYPE SHARED)
+        if (${pkg_name}_USE_STATIC_LIBS)
+            set(_LIB_SEARCH_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+            set(_LIB_TYPE STATIC)
+        endif ()
+        set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
+        find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH)
+        if (NOT ${_LIB_NAME}_LIB AND BUILD_LITE AND PLATFORM_ARM)
+            set(${_LIB_NAME}_LIB "${${pkg_name}_BASE_DIR}/${lib_path}/lib${_LIB_SEARCH_NAME}.so")
+        endif(NOT ${_LIB_NAME}_LIB AND BUILD_LITE AND PLATFORM_ARM)
+        if(NOT ${_LIB_NAME}_LIB)
+            return()
+        endif()
+
+        add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL)
+        if (WIN32 AND ${_LIB_TYPE} STREQUAL "SHARED")
+            set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_IMPLIB_RELEASE ${${_LIB_NAME}_LIB})
+        else()
+            set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${${_LIB_NAME}_LIB})
+        endif()
+
+        if (EXISTS ${${pkg_name}_BASE_DIR}/include)
+            set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES 
+                INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include")
+        endif ()
+
+        list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME})
+        message("found ${${_LIB_NAME}_LIB}")
+        STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB})
+        set(${pkg_name}_LIBPATH ${LIBPATH} CACHE STRING INTERNAL)
+    endforeach(_LIB_NAME)
+
+    set(${pkg_name}_LIBS ${${pkg_name}_LIBS} PARENT_SCOPE)
+endfunction()
+
+function(__exec_cmd)
+    set(options )
+    set(oneValueArgs WORKING_DIRECTORY)
+    set(multiValueArgs COMMAND)
+
+    cmake_parse_arguments(EXEC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
+
+    execute_process(COMMAND ${EXEC_COMMAND}
+            WORKING_DIRECTORY ${EXEC_WORKING_DIRECTORY}
+            RESULT_VARIABLE RESULT)
+    if(NOT RESULT EQUAL "0")
+        message(FATAL_ERROR "error! when ${EXEC_COMMAND} in ${EXEC_WORKING_DIRECTORY}")
+    endif()
+endfunction()
+
+function(__check_patches pkg_patches)
+    # check patches
+    if (PKG_PATCHES)
+        file(TOUCH ${_MS_LIB_CACHE}/${pkg_name}_patch.md5)
+        file(READ ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${pkg_name}_PATCHES_MD5)
+
+        message("patches md5:${${pkg_name}_PATCHES_MD5}")
+
+        set(${pkg_name}_PATCHES_NEW_MD5 )
+        foreach(_PATCH ${PKG_PATCHES})
+            file(MD5 ${_PATCH} _PF_MD5)
+            set(${pkg_name}_PATCHES_NEW_MD5 "${${pkg_name}_PATCHES_NEW_MD5},${_PF_MD5}")
+        endforeach(_PATCH)
+
+        if (NOT ${pkg_name}_PATCHES_MD5 STREQUAL ${pkg_name}_PATCHES_NEW_MD5)
+            set(${pkg_name}_PATCHES ${PKG_PATCHES})
+            file(REMOVE_RECURSE "${_MS_LIB_CACHE}/${pkg_name}-subbuild")
+            file(WRITE ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${${pkg_name}_PATCHES_NEW_MD5})
+            message("patches changed : ${${pkg_name}_PATCHES_NEW_MD5}")
+        endif ()
+    endif ()
+endfunction()
+
+set(MS_FIND_NO_DEFAULT_PATH NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH
+                            NO_CMAKE_BUILDS_PATH NO_CMAKE_PACKAGE_REGISTRY NO_CMAKE_SYSTEM_PATH
+                            NO_CMAKE_SYSTEM_PACKAGE_REGISTRY)
+set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
+function(mindspore_add_pkg pkg_name )
+
+    message("---------add pkg: " ${pkg_name} "---------")
+    set(options )
+    set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH CUSTOM_CMAKE)
+    set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES ONLY_MAKE ONLY_MAKE_INCS ONLY_MAKE_LIBS)
+    cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
+
+    if (NOT PKG_LIB_PATH)
+        set(PKG_LIB_PATH lib)
+    endif ()
+
+    if(NOT PKG_EXE)
+        set(PKG_EXE 0)
+    endif()
+
+    set(__FIND_PKG_NAME ${pkg_name})
+    string(TOLOWER ${pkg_name} pkg_name)
+    message("pkg name:${__FIND_PKG_NAME},${pkg_name}")
+
+    set(${pkg_name}_PATCHES_HASH )
+    foreach(_PATCH ${PKG_PATCHES})
+        file(MD5 ${_PATCH} _PF_MD5)
+        set(${pkg_name}_PATCHES_HASH "${${pkg_name}_PATCHES_HASH},${_PF_MD5}")
+    endforeach(_PATCH)
+
+    # check options
+    set(${pkg_name}_CONFIG_TXT
+            "${CMAKE_CXX_COMPILER_VERSION}-${CMAKE_C_COMPILER_VERSION}
+            ${ARGN} - ${${pkg_name}_USE_STATIC_LIBS}- ${${pkg_name}_PATCHES_HASH}
+            ${${pkg_name}_CXXFLAGS}--${${pkg_name}_CFLAGS}--${${pkg_name}_LDFLAGS}")
+    string(REPLACE ";" "-" ${pkg_name}_CONFIG_TXT ${${pkg_name}_CONFIG_TXT})
+    string(MD5 ${pkg_name}_CONFIG_HASH ${${pkg_name}_CONFIG_TXT})
+
+    message("${pkg_name} config hash: ${${pkg_name}_CONFIG_HASH}")
+
+    set(${pkg_name}_BASE_DIR ${_MS_LIB_CACHE}/${pkg_name}_${${pkg_name}_CONFIG_HASH})
+    set(${pkg_name}_DIRPATH ${${pkg_name}_BASE_DIR} CACHE STRING INTERNAL)
+
+    if(EXISTS ${${pkg_name}_BASE_DIR}/options.txt AND PKG_HEAD_ONLY)
+        set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
+        add_library(${pkg_name} INTERFACE)
+        target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
+        if (${PKG_RELEASE})
+            __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
+        endif ()
+        return()
+    endif ()
+
+    set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR})
+    set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE)
+
+    if (PKG_LIBS)
+        __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
+        if(${pkg_name}_LIBS)
+            set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
+            message("Found libs: ${${pkg_name}_LIBS}")
+            return()
+        endif()
+    elseif(NOT PKG_HEAD_ONLY)
+        find_package(${__FIND_PKG_NAME} ${PKG_VER} ${MS_FIND_NO_DEFAULT_PATH})
+        if (${__FIND_PKG_NAME}_FOUND)
+            set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
+            message("Found pkg: ${__FIND_PKG_NAME}")
+            return()
+        endif ()
+    endif ()
+
+    if (NOT PKG_DIR)
+        if (PKG_GIT_REPOSITORY)
+            __download_pkg_with_git(${pkg_name} ${PKG_GIT_REPOSITORY} ${PKG_GIT_TAG} ${PKG_MD5})
+        else()
+            __download_pkg(${pkg_name} ${PKG_URL} ${PKG_MD5})
+        endif()
+        foreach(_SUBMODULE_FILE ${PKG_SUBMODULES})
+            STRING( REGEX REPLACE "(.+)_(.+)" "\\1" _SUBMODEPATH ${_SUBMODULE_FILE})
+            STRING( REGEX REPLACE "(.+)/(.+)" "\\2" _SUBMODENAME ${_SUBMODEPATH})
+            file(GLOB ${pkg_name}_INSTALL_SUBMODULE ${_SUBMODULE_FILE}/*)
+            file(COPY ${${pkg_name}_INSTALL_SUBMODULE} DESTINATION ${${pkg_name}_SOURCE_DIR}/3rdparty/${_SUBMODENAME})
+        endforeach (_SUBMODULE_FILE)
+    else()
+        set(${pkg_name}_SOURCE_DIR ${PKG_DIR})
+    endif ()
+    file(WRITE ${${pkg_name}_BASE_DIR}/options.txt ${${pkg_name}_CONFIG_TXT})
+    message("${pkg_name}_SOURCE_DIR : ${${pkg_name}_SOURCE_DIR}")
+
+    foreach(_PATCH_FILE ${PKG_PATCHES})
+        get_filename_component(_PATCH_FILE_NAME ${_PATCH_FILE} NAME)
+        set(_LF_PATCH_FILE ${CMAKE_BINARY_DIR}/_ms_patch/${_PATCH_FILE_NAME})
+        configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF @ONLY)
+
+        message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_LF_PATCH_FILE}")
+        execute_process(COMMAND ${Patch_EXECUTABLE} -p1 INPUT_FILE ${_LF_PATCH_FILE}
+                WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}
+                RESULT_VARIABLE Result)
+        if(NOT Result EQUAL "0")
+            message(FATAL_ERROR "Failed patch: ${_LF_PATCH_FILE}")
+        endif()
+    endforeach(_PATCH_FILE)
+    foreach(_SOURCE_DIR ${PKG_SOURCEMODULES})
+        file(GLOB ${pkg_name}_INSTALL_SOURCE ${${pkg_name}_SOURCE_DIR}/${_SOURCE_DIR}/*)
+        file(COPY ${${pkg_name}_INSTALL_SOURCE} DESTINATION ${${pkg_name}_BASE_DIR}/${_SOURCE_DIR}/)
+    endforeach (_SUBMODULE_FILE)
+    file(LOCK ${${pkg_name}_BASE_DIR} DIRECTORY GUARD FUNCTION RESULT_VARIABLE ${pkg_name}_LOCK_RET TIMEOUT 600)
+    if(NOT ${pkg_name}_LOCK_RET EQUAL "0")
+        message(FATAL_ERROR "error! when try lock ${${pkg_name}_BASE_DIR} : ${${pkg_name}_LOCK_RET}")
+    endif()
+
+    if (PKG_CUSTOM_CMAKE)
+        file(GLOB ${pkg_name}_cmake ${PKG_CUSTOM_CMAKE}/CMakeLists.txt)
+        file(COPY ${${pkg_name}_cmake} DESTINATION ${${pkg_name}_SOURCE_DIR})
+    endif ()
+
+    if(${pkg_name}_SOURCE_DIR)
+        if (PKG_HEAD_ONLY)
+            file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*)
+            file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR})
+            set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
+            if (NOT PKG_RELEASE)
+                add_library(${pkg_name} INTERFACE)
+                target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
+            endif ()
+
+        elseif (PKG_ONLY_MAKE)
+            __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_CXXFLAGS} -j${THNUM}
+                    WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
+            set(PKG_INSTALL_INCS ${PKG_ONLY_MAKE_INCS})
+            set(PKG_INSTALL_LIBS ${PKG_ONLY_MAKE_LIBS})
+            file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS})
+            file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS})
+            file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include)
+            file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib)
+
+        elseif (PKG_CMAKE_OPTION)
+            # in cmake
+            file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
+            if (${pkg_name}_CFLAGS)
+                set(${pkg_name}_CMAKE_CFLAGS "-DCMAKE_C_FLAGS=${${pkg_name}_CFLAGS}")
+            endif ()
+            if (${pkg_name}_CXXFLAGS)
+                set(${pkg_name}_CMAKE_CXXFLAGS "-DCMAKE_CXX_FLAGS=${${pkg_name}_CXXFLAGS}")
+            endif ()
+
+            if (${pkg_name}_LDFLAGS)
+                if (${pkg_name}_USE_STATIC_LIBS)
+                    #set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_STATIC_LINKER_FLAGS=${${pkg_name}_LDFLAGS}")
+                else()
+                    set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_SHARED_LINKER_FLAGS=${${pkg_name}_LDFLAGS}")
+                endif ()
+            endif ()
+
+            __exec_cmd(COMMAND ${CMAKE_COMMAND} ${PKG_CMAKE_OPTION} -G ${CMAKE_GENERATOR}
+                    ${${pkg_name}_CMAKE_CFLAGS} ${${pkg_name}_CMAKE_CXXFLAGS} ${${pkg_name}_CMAKE_LDFLAGS}
+                    -DCMAKE_INSTALL_PREFIX=${${pkg_name}_BASE_DIR} ${${pkg_name}_SOURCE_DIR}/${PKG_CMAKE_PATH}
+                    WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
+
+            __exec_cmd(COMMAND ${CMAKE_COMMAND} --build . --target install -- -j${THNUM}
+                    WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
+
+        else()
+            if (${pkg_name}_CFLAGS)
+                set(${pkg_name}_MAKE_CFLAGS "CFLAGS=${${pkg_name}_CFLAGS}")
+            endif ()
+            if (${pkg_name}_CXXFLAGS)
+                set(${pkg_name}_MAKE_CXXFLAGS "CXXFLAGS=${${pkg_name}_CXXFLAGS}")
+            endif ()
+            if (${pkg_name}_LDFLAGS)
+                set(${pkg_name}_MAKE_LDFLAGS "LDFLAGS=${${pkg_name}_LDFLAGS}")
+            endif ()
+            # in configure && make
+            if (PKG_PRE_CONFIGURE_COMMAND)
+                __exec_cmd(COMMAND ${PKG_PRE_CONFIGURE_COMMAND}
+                        WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
+            endif ()
+
+            if (PKG_CONFIGURE_COMMAND)
+                __exec_cmd(COMMAND ${PKG_CONFIGURE_COMMAND}
+                        ${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS}
+                        --prefix=${${pkg_name}_BASE_DIR}
+                        WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
+            endif ()
+            set(${pkg_name}_BUILD_OPTION ${PKG_BUILD_OPTION})
+            if (NOT PKG_CONFIGURE_COMMAND)
+                set(${pkg_name}_BUILD_OPTION ${${pkg_name}_BUILD_OPTION}
+                        ${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS})
+            endif ()
+            # build
+            __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_BUILD_OPTION} -j${THNUM}
+                    WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
+
+            if (PKG_INSTALL_INCS OR PKG_INSTALL_LIBS)
+                file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS})
+                file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS})
+                file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include)
+                file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib)
+            else()
+                __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} install WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
+            endif ()
+        endif ()
+    endif()
+
+    if (PKG_LIBS)
+        __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
+        set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
+        if(NOT ${pkg_name}_LIBS)
+            message(FATAL_ERROR "Can not find pkg: ${pkg_name}")
+        endif()
+    else()
+        find_package(${__FIND_PKG_NAME} ${PKG_VER} QUIET ${MS_FIND_NO_DEFAULT_PATH})
+        if (${__FIND_PKG_NAME}_FOUND)
+            set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
+            message("Found pkg: ${${__FIND_PKG_NAME}_LIBRARIES}")
+            return()
+        endif ()
+    endif ()
+endfunction()

docs/GRPC.md

@@ -0,0 +1,167 @@
+# gRPC接口使用
+
+## 概述
+MindSpore Serving提供gRPC接口访问Serving服务。在Python环境下，我们提供[mindspore_serving.client](../mindspore_serving/client/python/client.py) 接口填写请求、解析回复。接下来我们详细说明`mindspore_serving.client`如何使用。
+
+## 样例
+在详细说明接口之前，我们先看几个样例。
+
+### add样例
+样例来源于[add example](../mindspore_serving/example/add/client.py)
+```
+from mindspore_serving.client import Client
+import numpy as np
+
+
+def run_add_common():
+    """invoke Servable add method add_common"""
+    client = Client("localhost", 5500, "add", "add_common")
+    instances = []
+
+    # instance 1
+    x1 = np.asarray([[1, 1], [1, 1]]).astype(np.float32)
+    x2 = np.asarray([[1, 1], [1, 1]]).astype(np.float32)
+    instances.append({"x1": x1, "x2": x2})
+
+    # instance 2
+    x1 = np.asarray([[2, 2], [2, 2]]).astype(np.float32)
+    x2 = np.asarray([[2, 2], [2, 2]]).astype(np.float32)
+    instances.append({"x1": x1, "x2": x2})
+
+    # instance 3
+    x1 = np.asarray([[3, 3], [3, 3]]).astype(np.float32)
+    x2 = np.asarray([[3, 3], [3, 3]]).astype(np.float32)
+    instances.append({"x1": x1, "x2": x2})
+
+    result = client.infer(instances)
+    print(result)
+
+
+if __name__ == '__main__':
+    run_add_common()
+```
+按照[入门流程](../README_CN.md/#%E5%BF%AB%E9%80%9F%E5%85%A5%E9%97%A8) 导出模型，启动Serving服务器，并执行客户端代码。当运行正常后，将打印以下结果，为了展示方便，格式作了调整：
+```
+[{'y': array([[2., 2.], [2., 2.]], dtype=float32)},
+ {'y': array([[4., 4.], [4., 4.]], dtype=float32)},
+ {'y': array([[6., 6.], [6., 6.]], dtype=float32)}]
+```
+
+以下将对其中的细节进行说明。
+1. 构造`Client`
+
+    构造`Client`时，指示Serving的ip和端口号，并给定Servable名称和它提供的方法。这里的Servable可以是单个模型，也可以是多个模型的组合，一个Servable可以提供多种方法以提供不同的服务。
+
+    上面的`add`样例， Serving运行在本地（`localhost`），指定的gRPC端口号为`5500`，运行了`add` Servable，`add` Servable提供了`add_common`方法。
+
+2. 添加实例
+
+   每次请求可包括一个或多个实例，每个实例之间相互独立，结果互不影响。
+
+   比如：`add` Servable提供的`add_common`方法提供两个2x2 Tensor相加功能，即一个实例包含两个2x2 Tensor输入，一个2x2 Tensor输出。一次请求可包括一个、两个或者多个这样的实例，针对每个实例返回一个结果。上述`add`样例提供了三个实例，预期将返回三个实例的结果。
+    ```
+    Given Request:
+    instance1:
+    x1 = [[1, 1], [1, 1]]
+    x2 = [[1, 1], [1, 1]]
+
+    instance2:
+    x1 = [[2, 2], [2, 2]]
+    x2 = [[2, 2], [2, 2]]
+
+    instance3:
+    x1 = [[3, 3], [3, 3]]
+    x2 = [[3, 3], [3, 3]]
+
+    Expected Relpy:
+    instance1:
+    y = [[2., 2.], [2., 2.]] # instance1 x1 + x2
+
+    instance2:
+    y = [[4., 4.], [4., 4.]] # instance2 x1 + x2
+
+    instance3:
+    y = [[6., 6.], [6., 6.]] # instance3 x1 + x2
+    ```
+
+   `Client.infer`接口入参可为实例的list、tuple或者单个实例。每个实例的输入由dict表示，dict的key即为输入的名称，value为输入的值。
+   
+   value可以是以下格式的值：
+
+    |  值类型   | 说明  |  举例  |
+    |  ----  | ----  |  ---- |
+    | numpy array  | 用以表示Tensor | np.ones((3,224), np.float32) |
+    | numpy scalar | 用以表示Scalar | np.int8(5)  |
+    | python bool int float | 用以表示Scalar, 当前int将作为int32, float将作为float32 | 32.0  |
+    | python str | 用以表示字符串 | "this is a text"  |
+    | python bytes | 用以表示二进制数据 | 图片数据  |
+    
+    上面的add样例，`add` Servable提供的`add_common`方法入参名为`x1`和`x2`，添加每个实例时指定每个输入的值。
+
+3. 获取推理结果
+    通过`Client.infer`填入一个或多个实例。
+    返回可能有以下形式：
+    
+    所有实例推理正确：
+
+    ```
+    [{'y': array([[2., 2.], [2., 2.]], dtype=float32)},
+     {'y': array([[4., 4.], [4., 4.]], dtype=float32)},
+     {'y': array([[6., 6.], [6., 6.]], dtype=float32)}]
+    ```
+
+    针对所有实例共同的错误，返回一个包含`error`的dict。将例子中Client构造时填入的`add_common`改为`add_common2`，将返回结果：
+
+    ```
+    {'error', 'Request Servable(add) method(add_common2), method is not available'}
+    ```
+
+    部分实例推理错误，出错的推理实例将返回包含`error`的dict。将instance2一个输入的`dtype`改为`np.int32`，将返回结果：
+
+    ```
+    [{'y': array([[2., 2.], [2., 2.]], dtype=float32)},
+     {'error': 'Given model input 1 data type kMSI_Int32 not match ...'},
+     {'y': array([[6., 6.], [6., 6.]], dtype=float32)}]
+    ```
+    每个实例返回一个dict，key的值来自于Servable的方法定义，例如本例子中，`add` Servable提供的`add_common`方法输出仅有一个，为`y`。value为以下格式：
+
+    |  Serving输出类型 | Client返回类型   | 说明  |  举例  |
+    |  ----  | ----  |  ---- | ---- |
+    | Tensor | numpy array | tensor array | np.ones((3,224), np.float32) |
+    | Scalar: <br>int8, int16, int32, int64, <br>uint8, uint16, uint32, uint64, <br>bool, float16, float32, float64 | numpy scalar | Scalar格式的数据转为numpy scalar | np.int8(5)  |
+    | String | python str | 字符串格式输出转为python str | "news_car"  |
+    | Bytes | python bytes | 二进制格式输出转为python bytes | 图片数据  |
+
+
+### lenet样例
+样例来源于[lenet example](../mindspore_serving/example/lenet/client.py) 。通过lenet样例来说明二进制的输入，lenet输入为二进制，输出为Scalar。
+```
+import os
+from mindspore_serving.client import Client
+
+
+def run_predict():
+    client = Client("localhost", 5500, "lenet", "predict")
+    instances = []
+    for path, _, file_list in os.walk("./test_image/"):
+        for file_name in file_list:
+            image_file = os.path.join(path, file_name)
+            print(image_file)
+            with open(image_file, "rb") as fp:
+                instances.append({"image": fp.read()})
+    result = client.infer(instances)
+    if "error" in result:
+        print("error happen:", result["error"])
+        return
+    print(result)
+
+
+if __name__ == '__main__':
+    run_predict()
+```
+上面lenet例子中，输入的每个实例的唯一的输入`image`为文件二进制方式读取的bytes。
+正常结束执行后，可能将会有以下打印：
+```
+[{'result': 4}, {'result': 4}, {'result': 4}, {'result': 4}]
+```
+

docs/MODEL.md

@@ -0,0 +1,147 @@
+# 模型配置接口
+
+## 概述
+
+MindSpore Serving的Servable包含两种类型。一种是推理服务来源于单模型，一种是推理服务来源于多模型组合，多模型组合当前不支持。
+
+本文将说明如何对单模型进行配置以提供Servable，以下所有Servable配置说明针对的是单模型Servable，Serving客户端简称客户端。
+
+## 相关概念
+
+### 预处理和后处理
+
+模型提供推理能力，模型的每个输入和输出的数据类型、数据长度、Shape是固定的。
+
+如果客户端发来的数据不能直接满足模型输入要求，需要通过预处理转化为满足模型输入的数据。
+如果模型的输出不直接提供给客户端，需要通过后处理转化为所需的输出数据。
+
+![image](image/resnet_example.png)
+
+比如，针对Resnet推理模型，客户端发来的数据为jpg、png等格式的图片，预期返回图片的分类。Resnet模型输入为经过图片`Decode`、`Resize`、`Normalize`等操作产生的Tensor，输出为每个类别的得分Tensor。需要通过预处理将图片转化为满足模型输入的Tensor，通过后处理返回得分最大的类别名称或者前5类别名称及其得分。
+
+在不同的场景下，如果来自客户端的数据输入组成、结构或类型不同，可以提供不同的预处理。如果对模型的输出也有不同的要求，可以提供不同的后处理。比如上述`resnet` Servable，针对返回`得分最大的类别名称`还是`前5类别名称及其得分`这两种场景提供了两个后处理。
+
+### 方法
+
+上述的`resnet` Servable提供了`classify_top5`和`classify_top1`两个方法（`Method`）。`classify_top5`输入为`image`，输出为`label`和`score`，返回前5的分类名称和得分。`classify_top1`预处理和`classify_top5`一致，而后处理不同，输入为`image`，输出为`label`，返回最大得分的分类名称。
+
+一个Servable可提供一个或多个方法，Servable的名称和方法的名称标记了Serving提供的一个服务，每个方法对客户端提供的数据进行可选的预处理，接着进行模型推理，对模型的推理结果进行可选的后处理，最后将需要的结果返回给客户端。
+
+即，每个方法：
+
+- 指定可选的预处理和可选的后处理；
+- 定义方法输入、预处理、模型、后处理、方法输出之间的数据流，前者可作为后者的输入。比如方法输出的值可来源于方法输入、预处理、模型或后处理；
+- 指定方法名，使客户端可以通过方法名指定使用的方法；
+- 指定方法的输入和输出名称，使客户端可以通过名称来指定输入、获取输出。
+
+### 实例
+
+每次请求可包括一个或多个实例，每个实例之间相互独立，结果互不影响。比如一张图片返回一个分类类别，三张独立的图片独立返回三个分类类别。
+
+## 模型配置
+
+以Resnet模型为例，模型配置文件目录结果如下图所示：
+
+<pre><font color="#268BD2"><b>resnet/</b></font>
+├── <font color="#268BD2"><b>1</b></font>
+│   └── resnet_classify.minir
+├── <font color="#268BD2"><b>2</b></font>
+│   └── resnet_classify.minir
+└── servable_config.py
+</pre>
+
+目录`resnet`指示Servable的名称。
+
+通过`servable_config.py`配置Servable，其中包括预处理和后处理定义、模型声明、方法定义。
+
+目录`1`和`2`表示版本`1`和版本`2`的模型，模型版本为正整数，从`1`开始，数字越大表示版本越新。
+
+### 预处理和后处理定义
+
+预处理和后处理定义方式例子如下：
+
+```
+def resnet_preprocess(instances): # instances: tuple of instance
+    for instance in instances:   # instance: tuple of input
+        image = instance[0] # input 0
+        # input1 = instance[1] # input 1 if exist
+
+        # Decode, Resize, Normalize
+        image_result = ...
+        yield (image_result,)
+
+idx_to_label = ["person", "car", ...]
+def resnet_postprocess_top1(instances):
+    for instance in instances:
+        score = instance[0] # get input 0
+        max_idx = np.argmax(score)
+        yield (idx_to_label[max_idx],)
+
+def resnet_postprocess_top5(instances):
+    for instance in instances:
+        score = instance[0] # get input 0
+        ids = np.argsort(score)[:5] # top 5
+        ret_label = idx_to_label[ids]
+        ret_score = score[ids]
+        yield (",".join(ret_label), ret_score)
+```
+
+预处理和后处理定义格式相同，入参为实例数据组成的tuple，每个实例数据为输入数据组成的tuple，每个输入数据为**numpy对象**，通过`yield`返回实例的处理结果，`yield`返回的数据类型可为**numpy对象、Python的bool、int、float、str、bytes**组成的tuple。
+
+预处理和后处理输入的来源和输出的使用由[方法定义](#方法定义)决定。
+
+### 模型声明
+
+```
+from mindspore_serving.worker import register
+register.declare_servable(servable_file="resnet_classify.minir", model_format="MindIR", with_batch_dim=True)
+```
+其中`declare_servable`入参`servable_file`指示模型的文件名称；`model_format`指示模型的模型类别，当前Ascend310环境支持`OM`和`MindIR`两种模型类型，Ascend910环境仅支持`MindIR`模型类型。
+
+如果模型输入输出第1维度不是`batch`维度，需要设置参数`with_batch_dim=False`，`with_batch_dim`默认为`True`。
+
+`with_batch_dim`为`True`，主要针对处理图片、文本等包含`batch`维度的模型。假设`batch_size=2`，当前请求有3个实例，共3张图片，会拆分为2次模型推理，第1次处理2张图片返回2个结果，第2次对剩余的1张图片进行拷贝做一次推理并返回1个结果，最终返回3个结果。
+
+![image](image/resnet_with_batch.png)
+
+`with_batch_dim`为`False`，主要针对不涉及或不考虑`batch`维度的模型。比如输入输出为二维Tensor的矩阵乘模型。请求的每个实例将单独作一次推理任务。
+
+![image](./image/matmul_without_batch.png)
+
+### 方法定义
+
+方法定义的例子如下：
+
+```
+@register.register_method(output_names=["label"])
+def classify_top1(image):
+    x = register.call_preprocess(resnet_preprocess, image)
+    x = register.call_servable(x)
+    x = register.call_postprocess(resnet_postprocess_top1, x)
+    return x
+
+@register.register_method(output_names=["label", "score"])
+def classify_top5(image):
+    x = register.call_preprocess(resnet_preprocess, image)
+    x = register.call_servable(x)
+    label, score = register.call_postprocess(resnet_postprocess_top5, x)
+    return label, score
+```
+
+Python函数和Servable方法对应关系如下表：
+
+|  Python函数 | Servable方法  |
+|  ----  | ----  |
+| 函数名 | 方法名 |
+| 入参和入参名称 | 入参和入参名称 |
+| `register_method`的`output_names`参数 | 出参和出参名称 |
+
+`call_preprocess`指示了使用的预处理及其输入。
+
+`call_servable`指示了模型推理的输入。
+
+`call_postprocess`指示了使用的后处理及其输入。
+
+`return`指示了方法的返回数据，和`register_method`的`output_names`参数对应。
+
+方法定义不能包括if、for、while等分支结构，预处理和后处理可选，模型推理必选，且顺序不能打乱。

docs/RESTful.md

@@ -0,0 +1,231 @@
+
+
+## RESTful 接口使用说明
+
+MindSpore Serving支持`GPRC`和`RESTful`两种请求方式。本章节介绍`RESTful`类型请求。
+
+部署`Serving`参考[快速入门](https://gitee.com/mindspore/serving/blob/master/README_CN.md#快速入门) 章节。
+
+与通过`master.start_grpc_server("127.0.0.1", 5500)`启动`GRPC`服务不同的是，`RESTful`服务需要通过`master.start_restful_server("0.0.0.0", 1500)`来启动。
+
+### 1.请求方式
+
+当前支持`POST`类型的RESTful请求，请求格式如下：
+
+```
+POST http://${HOST}:${PORT}/model/${MODLE_NAME}[/version/${VERSION}]:${METHOD_NAME}
+```
+
+其中：
+
+- `HOST`：指定访问的IP地址；
+- `PORT`：指定访问的端口号；
+- `MODEL_NAME`：请求的模型名称；
+- `VERSION`：表示版本号。版本号是可选的，若未指定具体版本号，则默认使用模型的最新版本。
+- `METHOD_NAME`：表示请求模型的具体方法名称。
+
+如果使用`curl`工具，RESTful请求方式如下：
+
+```
+curl -X POST -d '${REQ_JSON_MESSAGE}' http://${HOST}:${PORT}/model/${MODLE_NAME}[/version/${VERSION}]:${METHOD_NAME}
+```
+
+例子：请求`lenet`模型的`predict`方法进行数字图片的推理，请求如下：
+
+```
+curl -X POST -d '{"instances":{"image":{"b64":"babe64-encoded-string"}' http://127.0.0.1:1500/model/lenet/version/1:predict
+```
+
+其中：`babe64-encoded-string`是数字`1`图片经过`base64`编码之后的字符串。由于字符串比较长，不显式列出。
+
+
+
+### 2.请求输入格式
+
+RESTful支持`Json`请求格式，`key`固定为`instances`，`value`:表示多个实例。
+
+每个实例通过`key-value`格式的`Json`表示。其中：
+
+- `key`：表示输入名称，需要与请求模型提供的方法的输入参数名称一致，若不一致，则请求失败。
+
+- `value`：表示具体的值。当前支持的`value`类型：
+
+  - 标量：`str`、`bytes`、`int`、`float`、`bool`；
+
+    `bytes`：通过`base64`编码方式支持。
+
+  - 张量：`int`、`float`、`bool`。
+
+    张量通过数组格式表示数据和维度信息。
+
+`Json`中支持的`int`类型：是`int32`表示的范围，`float`类型：是`float32`表示的范围。
+
+请求格式：
+
+```
+{
+"instances":[
+    {
+        "input_name1":<value>|<list>|<object>,
+        "input_name2":<value>|<list>|<object>,
+        ...
+    },
+    {
+    	"input_name1":<value>|<list>|<object>,
+        "input_name2":<value>|<list>|<object>,
+        ...
+    }
+    ...
+]
+}
+```
+
+例子：
+
+```
+{
+    "instances":[
+        {
+            "tag":"one",
+            "box":[[1,1],[2,3],[3,4]]
+            "image":{"b64":"iVBOR...ggg==="}
+        },
+        {
+            "tag":"two"
+            "box":[[2,2],[5,5],[6,6]]
+            "image":{"b64":"iVBOR...QmCC", "type":"bytes"}
+        }
+    ]
+}
+```
+
+其中：`iVBOR...ggg===`是图片数字`0`经过`base64`编码之后的省略字符串。`iVBOR...QmCC`是图片数字`1`经过`base64`编码之后的省略字符串。不同图片编码出来的字符串可能不同，上述是示意说明。
+
+`bytes`类型需要通过`base64`编码进行表示。`base64`除了支持`bytes`类型，也支持表示其他标量和张量，此时需要通过`type`指定数据类型，通过`shape`指定维度信息。
+
+- `type`：可选，如果不指定，默认为`bytes`；
+
+  支持`int8`、`int16`、`int32`、`int64`、`uint8`、`uint16`、`uint32`、`uint64`、`fp16`、`fp32`、`fp64`、`bool`、`str`、`bytes`.　
+
+- `shape`：可选，如果不指定，默认为`[1]`.
+
+例子：
+
+如果要用`base64`编码表示：`int16`的数据类型，`shape`为3*2，值是`[[1,1],[2,3],[3,4]]`的张量，则表示如下：
+
+```json
+{
+    "instances":[
+        {
+            "tag":"one",
+            "box":{"b64":"AQACAAIAAwADAAQA", "type":"int16", "shape":[3,2]},
+            "image":{"b64":"iVBOR...ggg==="}
+        }
+    ]
+}
+```
+
+其中`AQACAAIAAwADAAQA`：是`[[1,1],[2,3],[3,4]]`经过`base64`编码字后的字符串。
+
+
+
+##### 支持的类型总结如下：
+
+|                          支持的类型                          | 例子                                                         | 备注                               |
+| :----------------------------------------------------------: | ------------------------------------------------------------ | ---------------------------------- |
+|                            `int`                             | 1，[1,2,3,4]                                                 | 默认`int32`表示范围                |
+|                           `float`                            | 1.0，[[1.2, 2.3], [3.0, 4.5]]                                | 默认`float32`表示范围              |
+|                            `bool`                            | true，false，[[true],[false]]                                | `bool`类型                         |
+|                           `string`                           | "hello"或者<br/>  {"b64":"aGVsbG8=", "type":"str"}           | 直接表示或者指定`type`方式表示     |
+|                           `bytes`                            | {"b64":"AQACAAIAAwADAAQA"} 或者 <br>{"b64":"AQACAAIAAwADAAQA", "type":"bytes"} | 如果不填`type`，默认为`bytes`      |
+| `int8`,`int16`,`int32`,`int64`,`uint8`,`uint16`,`uint32`,`uint64` `float16`,`float32`,`bool` | {"b64":"AQACAAIAAwADAAQA", "type":"int16", "shape":[3,2]}    | 利用base64编码，表示指定type的数据 |
+
+### 3.请求应答格式
+
+应答格式与请求格式保持一致。返回`Json`格式信息。应答格式如下：
+
+```
+{
+"instances":[
+    {
+        "output_name1":<value>|<list>|<object>,
+        "output_name2":<value>|<list>|<object>,
+        ...
+    },
+    {
+    	"output_name1":<value>|<list>|<object>,
+        "output_name2":<value>|<list>|<object>,
+        ...
+    }
+    ...
+]
+}
+```
+
+
+
+1. 多实例请求后，如果多实例全部成功处理，则响应格式如下：
+
+   例子：`lenet`请求识别数字`0`和数字`1`
+
+   ```json
+   {
+       "instances":[
+           {
+               "result":0
+           },
+           {	
+               "result":1
+           }
+   	]
+   }
+   ```
+
+   
+
+2. 如果部分实例出错，则响应格式如下：
+
+   例子：`lenet`请求识别数字`0`和一个错误数字图片
+
+   ```json
+   {
+       "instances":[
+           {
+               "result":0
+           },
+           {
+               "error_msg":"Preprocess Failed"
+           }
+   	]
+   }
+   ```
+   
+   
+   
+3. 如果请求全部失败，则响应格式如下：
+
+   例子：`lenet`请求识别两张错误数字图片为例
+
+```json
+{
+    "instances":[
+        {
+            "error_msg":"Preprocess Failed"
+        },
+        {
+            "error_msg":"Time out"
+        }
+    ]
+}
+```
+
+4. 出现系统性或者其他解析等错误，则返回格式：
+
+   例子：`lenet`传入非法`Json`字符串
+
+```json
+{
+    "error_msg":"Parse request failed"
+}
+```
+

mindspore_serving/CMakeLists.txt

@@ -0,0 +1,137 @@
+# This branch assumes that gRPC and all its dependencies are already installed
+# on this system, so they can be located by find_package().
+
+# Find Protobuf installation
+# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib")
+
+add_library(protobuf::libprotobuf ALIAS protobuf::protobuf)
+add_executable(protobuf::libprotoc ALIAS protobuf::protoc)
+
+set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+if (CMAKE_CROSSCOMPILING)
+    find_program(_PROTOBUF_PROTOC protoc)
+else ()
+    set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+endif ()
+
+# Find gRPC installation
+# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
+if (EXISTS ${grpc_ROOT}/lib64)
+    set(gRPC_DIR "${grpc_ROOT}/lib64/cmake/grpc")
+else ()
+    set(gRPC_DIR "${grpc_ROOT}/lib/cmake/grpc")
+endif ()
+message("serving using grpc_DIR : " ${gPRC_DIR})
+
+find_package(gRPC CONFIG REQUIRED)
+message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+set(_GRPC_GRPCPP gRPC::grpc++)
+set(_REFLECTION gRPC::grpc++_reflection)
+
+if (CMAKE_CROSSCOMPILING)
+    find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+    find_program(_GRPC_PYTHON_PLUGIN_EXECUTABLE grpc_python_plugin)
+else ()
+    set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+    set(_GRPC_PYTHON_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_python_plugin>)
+endif ()
+
+# Proto file
+
+# Generated sources
+file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/proto/")
+file(GLOB_RECURSE PROTO_FILE_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/proto/ *.proto)
+foreach (proto_file ${PROTO_FILE_LIST})
+    message(------proto file: ${proto_file})
+    get_filename_component(proto_I_DIR "../" ABSOLUTE)
+    get_filename_component(proto_file_absolute "proto/${proto_file}" ABSOLUTE)
+    set(proto_file_relative "mindspore_serving/proto/${proto_file}")
+    string(REGEX REPLACE .proto "" proto_file_prefix ${proto_file})
+
+    set(protoc_output_prefix ${CMAKE_CURRENT_BINARY_DIR}/mindspore_serving/proto)
+    set(hw_proto_srcs "${protoc_output_prefix}/${proto_file_prefix}.pb.cc")
+    set(hw_proto_hdrs "${protoc_output_prefix}/${proto_file_prefix}.pb.h")
+    set(hw_grpc_srcs "${protoc_output_prefix}/${proto_file_prefix}.grpc.pb.cc")
+    set(hw_grpc_hdrs "${protoc_output_prefix}/${proto_file_prefix}.grpc.pb.h")
+    set(hw_py_pb2 "${protoc_output_prefix}/${proto_file_prefix}_pb2.py")
+    set(hw_py_pb2_grpc "${protoc_output_prefix}/${proto_file_prefix}_pb2_grpc.py")
+    add_custom_command(
+            OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}" "${hw_py_pb2}" "${hw_py_pb2_grpc}"
+            WORKING_DIRECTORY ${proto_I_DIR}
+            COMMAND ${_PROTOBUF_PROTOC}
+            ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+            --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+            -I "${proto_I_DIR}"
+            --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+            "${proto_file_relative}"
+            COMMAND ${_PROTOBUF_PROTOC}
+            ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+            --python_out "${CMAKE_CURRENT_BINARY_DIR}"
+            -I "${proto_I_DIR}"
+            --plugin=protoc-gen-grpc="${_GRPC_PYTHON_PLUGIN_EXECUTABLE}"
+            "${proto_file_relative}"
+            DEPENDS "${proto_file_absolute}")
+
+    list(APPEND PROTO_SRC_LIST ${hw_proto_srcs} ${hw_grpc_srcs})
+endforeach (proto_file)
+
+add_library(PROTO_SRC_LIB STATIC ${PROTO_SRC_LIST})
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}/mindspore_serving") # for proto header file
+include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}" "ccsrc")
+
+# serving_common for c++ server and python interface
+file(GLOB_RECURSE MASTER_CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+        "ccsrc/master/*.cc" "ccsrc/common/*.cc" "ccsrc/worker/*.cc" "ccsrc/inference/*.cc")
+
+file(GLOB_RECURSE RMV_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+        "ccsrc/master/version_control/*.cc")
+
+#list(REMOVE_ITEM MASTER_CORE_SRC_LIST ${RMV_SRC_LIST})
+
+list(APPEND SERVING_SRC ${MASTER_CORE_SRC_LIST})
+add_library(serving_common SHARED ${SERVING_SRC})
+
+include(CheckPIESupported)
+check_pie_supported()
+set_property(TARGET serving_common PROPERTY POSITION_INDEPENDENT_CODE TRUE)
+
+target_link_libraries(serving_common PRIVATE PROTO_SRC_LIB mindspore_serving::event mindspore_serving::event_pthreads)
+target_link_libraries(serving_common PRIVATE ${_REFLECTION} ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} pthread)
+
+# python
+add_compile_definitions(ENABLE_PYTHON)
+file(GLOB_RECURSE PY_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ccsrc/python/*.cc")
+
+find_package(Python3 3.7 COMPONENTS Interpreter Development)
+if (Python3_FOUND)
+    set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}")
+    set(PYTHON_LIBRARIES "${Python3_LIBRARIES}")
+else ()
+    find_python_package(py_inc py_lib)
+    set(PYTHON_INCLUDE_DIRS "${py_inc}")
+    set(PYTHON_LIBRARIES "${py_lib}")
+endif ()
+
+include_directories(${PYTHON_INCLUDE_DIRS})
+pybind11_add_module(_mindspore_serving ${PY_SRC_LIST})
+target_link_libraries(_mindspore_serving PRIVATE "${PYTHON_LIBRARIES}")
+target_include_directories(_mindspore_serving PRIVATE ${pybind11_INCLUDE_DIRS})
+target_link_libraries(_mindspore_serving PRIVATE serving_common)
+set_property(TARGET _mindspore_serving PROPERTY POSITION_INDEPENDENT_CODE TRUE)
+
+# user set path
+
+if (MS_WHL_LIB_PATH)
+    include_directories(${MS_WHL_LIB_PATH}/../include)
+    include_directories(${MS_WHL_LIB_PATH}/../)
+    target_link_libraries(serving_common PRIVATE ${MS_WHL_LIB_PATH}/libmindspore.so)
+elseif (MS_BACKEND)
+    include_directories(${MS_SOURCE_DIR}/build/package/mindspore)
+    include_directories(${MS_SOURCE_DIR}/build/package/mindspore/include)
+    target_link_libraries(serving_common PRIVATE ${MS_SOURCE_DIR}/build/package/mindspore/lib/libmindspore.so)
+else ()
+    message(FATAL_ERROR "Please check MindSpore path.")
+endif ()

mindspore_serving/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

mindspore_serving/ccsrc/common/buffer_tensor.cc

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/buffer_tensor.h"
+namespace mindspore::serving {
+
+BufferTensor::BufferTensor(DataType type, const std::vector<int64_t> shape, uint8_t *data, size_t data_len,
+                           bool data_readonly) {
+  set_shape(shape);
+  set_data_type(type);
+  data_ = data;
+  data_len_ = data_len;
+  data_readonly_ = data_readonly;
+}
+
+std::vector<int64_t> BufferTensor::shape() const { return shape_; }
+
+void BufferTensor::set_shape(const std::vector<int64_t> &shape) { shape_ = shape; }
+
+DataType BufferTensor::data_type() const { return type_; }
+
+void BufferTensor::set_data_type(DataType type) { type_ = type; }
+
+const uint8_t *BufferTensor::data() const { return data_; }
+
+size_t BufferTensor::data_size() const { return data_len_; }
+
+uint8_t *BufferTensor::mutable_data() {
+  if (data_readonly_) {
+    MSI_LOG_EXCEPTION << "Buffer tensor is create readonly";
+  }
+  return data_;
+}
+
+size_t BufferTensor::bytes_data_size() const {
+  if (!is_bytes_val_data()) {
+    return 0;
+  }
+  return 1;
+}
+
+void BufferTensor::get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const {
+  if (!is_bytes_val_data()) {
+    MSI_LOG_EXCEPTION << "Buffer tensor data type is not kMSI_Bytes or kMSI_String, cannot get bytes data";
+  }
+  data = data_;
+  bytes_len = data_len_;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/buffer_tensor.h

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_BUFFER_TENSOR_H
+#define MINDSPORE_BUFFER_TENSOR_H
+
+#include <vector>
+#include "common/serving_common.h"
+
+namespace mindspore::serving {
+
+class BufferTensor : public TensorBase {
+ public:
+  // the data's lifetime must longer than this object
+  BufferTensor(DataType type, std::vector<int64_t> shape, uint8_t *data, size_t data_len, bool data_readonly);
+  ~BufferTensor() = default;
+
+  // For all data type
+  std::vector<int64_t> shape() const override;
+  void set_shape(const std::vector<int64_t> &shape) override;
+  DataType data_type() const override;
+  void set_data_type(DataType type) override;
+
+  // All the following interfaces are not for kMSI_String and kMSI_Bytes
+  const uint8_t *data() const override;
+  size_t data_size() const override;
+  bool resize_data(size_t data_len) override { MSI_LOG_EXCEPTION << "Buffer tensor cannot resize data"; }
+  uint8_t *mutable_data() override;
+
+  // For kMSI_String and kMSI_Bytes
+  void clear_bytes_data() override { MSI_LOG_EXCEPTION << "Buffer tensor cannot clear bytes data"; }
+  void add_bytes_data(const uint8_t *data, size_t bytes_len) override {
+    MSI_LOG_EXCEPTION << "Buffer tensor cannot add bytes data";
+  }
+
+  size_t bytes_data_size() const override;
+  void get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const override;
+
+ private:
+  uint8_t *data_ = nullptr;
+  size_t data_len_ = 0;
+  std::vector<int64_t> shape_;
+  DataType type_;
+  bool data_readonly_ = false;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_BUFFER_TENSOR_H

mindspore_serving/ccsrc/common/exit_handle.cc

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/exit_handle.h"
+#include <signal.h>
+#include <utility>
+
+namespace mindspore {
+namespace serving {
+
+ExitHandle &ExitHandle::Instance() {
+  static ExitHandle instance;
+  return instance;
+}
+
+void ExitHandle::InitSignalHandle() {
+  if (!has_inited_.test_and_set()) {
+    signal(SIGINT, HandleSignal);
+    signal(SIGTERM, HandleSignal);
+  }
+}
+
+void ExitHandle::MasterWait() {
+  InitSignalHandle();
+  auto exit_future = master_exit_requested_.get_future();
+  exit_future.wait();
+}
+
+void ExitHandle::WorkerWait() {
+  InitSignalHandle();
+  auto exit_future = worker_exit_requested_.get_future();
+  exit_future.wait();
+}
+
+void ExitHandle::Stop() { HandleSignal(0); }
+
+bool ExitHandle::HasStopped() { return is_exit_; }
+
+void ExitHandle::HandleSignal(int sig) {
+  auto &instance = Instance();
+  if (!instance.has_exited_.test_and_set()) {
+    instance.master_exit_requested_.set_value();
+    instance.worker_exit_requested_.set_value();
+    instance.is_exit_.store(true);
+  }
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/common/exit_handle.h

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_EXIT_HANDLE_H
+#define MINDSPORE_SERVING_EXIT_HANDLE_H
+#include <functional>
+#include <atomic>
+#include <future>
+#include "common/serving_common.h"
+
+namespace mindspore {
+namespace serving {
+
+class MS_API ExitHandle {
+ public:
+  static ExitHandle &Instance();
+  void InitSignalHandle();
+  void MasterWait();
+  void WorkerWait();
+  void Stop();
+  bool HasStopped();
+
+ private:
+  std::promise<void> master_exit_requested_;
+  std::promise<void> worker_exit_requested_;
+  std::atomic_flag has_exited_ = ATOMIC_FLAG_INIT;
+  std::atomic_flag has_inited_ = ATOMIC_FLAG_INIT;
+  std::atomic<bool> is_exit_ = false;
+
+  static void HandleSignal(int sig);
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVING_EXIT_HANDLE_H

mindspore_serving/ccsrc/common/file_system_operation.cc

@@ -0,0 +1,72 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "common/file_system_operation.h"
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <algorithm>
+#include <ctime>
+#include "common/serving_common.h"
+
+namespace mindspore {
+namespace serving {
+bool DirOrFileExist(const std::string &file_path) {
+  int ret = access(file_path.c_str(), 0);
+  return (ret == -1) ? false : true;
+}
+
+std::vector<std::string> GetAllSubDirs(const std::string &dir_path) {
+  std::vector<std::string> SubDirs = GetAllSubDirsNotFullPath(dir_path);
+  for (auto &item : SubDirs) {
+    item = dir_path + "/" + item;
+  }
+  return SubDirs;
+}
+
+std::vector<std::string> GetAllSubDirsNotFullPath(const std::string &dir_path) {
+  DIR *dir = nullptr;
+  struct dirent *ptr = nullptr;
+  std::vector<std::string> SubDirs;
+
+  if ((dir = opendir(dir_path.c_str())) == NULL) {
+    MSI_LOG(ERROR) << "Open " << dir_path << " error!";
+    return std::vector<std::string>();
+  }
+
+  while ((ptr = readdir(dir)) != NULL) {
+    std::string name = ptr->d_name;
+    if (name == "." || name == "..") {
+      continue;
+    }
+    if (ptr->d_type == DT_DIR) {
+      SubDirs.push_back(name);
+    }
+  }
+  closedir(dir);
+  std::sort(SubDirs.begin(), SubDirs.end());
+  return SubDirs;
+}
+
+time_t GetModifyTime(const std::string &file_path) {
+  struct stat info;
+  (void)stat(file_path.c_str(), &info);
+  return info.st_mtime;
+}
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/common/file_system_operation.h

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_
+#define MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_
+
+#include <string>
+#include <vector>
+#include <ctime>
+
+namespace mindspore {
+namespace serving {
+char *ReadFile(const char *file, size_t *size);
+bool DirOrFileExist(const std::string &file_path);
+std::vector<std::string> GetAllSubDirs(const std::string &dir_path);
+std::vector<std::string> GetAllSubDirsNotFullPath(const std::string &dir_path);
+time_t GetModifyTime(const std::string &file_path);
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // !MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_

mindspore_serving/ccsrc/common/grpc_server.cc

@@ -0,0 +1,68 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/grpc_server.h"
+
+namespace mindspore::serving {
+
+Status GrpcServer::Start(std::shared_ptr<grpc::Service> service, const std::string &ip, uint32_t grpc_port,
+                         int max_msg_mb_size, const std::string &server_tag) {
+  service_ = service;
+  Status status;
+  if (in_running_) {
+    return INFER_STATUS_LOG_ERROR(SYSTEM_ERROR) << "Grpc server is running";
+  }
+
+  std::string server_address = ip + ":" + std::to_string(grpc_port);
+
+  grpc::EnableDefaultHealthCheckService(true);
+  grpc::reflection::InitProtoReflectionServerBuilderPlugin();
+  // Set the port is not reuseable
+  auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0);
+  grpc::ServerBuilder serverBuilder;
+  serverBuilder.SetOption(std::move(option));
+  if (max_msg_mb_size > 0) {
+    serverBuilder.SetMaxReceiveMessageSize(static_cast<int>(max_msg_mb_size * (1u << 20)));
+  }
+  serverBuilder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
+  serverBuilder.RegisterService(service.get());
+  server_ = serverBuilder.BuildAndStart();
+
+  if (server_ == nullptr) {
+    return INFER_STATUS_LOG_ERROR(FAILED)
+           << "Serving Error: create grpc server failed, gRPC address " << server_address;
+  }
+
+  auto grpc_server_run = [this, server_address, server_tag]() {
+    MSI_LOG(INFO) << server_tag << " start success,  listening on " << server_address;
+    std::cout << "Serving: " << server_tag << " start success, listening on " << server_address << std::endl;
+    server_->Wait();
+  };
+
+  grpc_thread_ = std::thread(grpc_server_run);
+  in_running_ = true;
+  return SUCCESS;
+}
+
+void GrpcServer::Stop() {
+  if (in_running_) {
+    server_->Shutdown();
+    grpc_thread_.join();
+  }
+  in_running_ = false;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/grpc_server.h

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_GRPC_SERVER_H
+#define MINDSPORE_GRPC_SERVER_H
+
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/health_check_service_interface.h>
+#include <grpcpp/ext/proto_server_reflection_plugin.h>
+#include <memory>
+#include <utility>
+#include <string>
+#include <future>
+#include "common/serving_common.h"
+
+namespace mindspore::serving {
+
+constexpr int gRpcDefaultMsgMBSize = 100;
+constexpr int gRpcMaxMBMsgSize = 512;  // max 512 MB
+
+class MS_API GrpcServer {
+ public:
+  GrpcServer() = default;
+  ~GrpcServer() { Stop(); }
+
+  Status Start(std::shared_ptr<grpc::Service> service, const std::string &ip, uint32_t grpc_port, int max_msg_size,
+               const std::string &server_tag);
+  void Stop();
+
+ private:
+  std::unique_ptr<grpc::Server> server_;
+  std::thread grpc_thread_;
+  bool in_running_ = false;
+  std::shared_ptr<grpc::Service> service_;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_GRPC_SERVER_H

mindspore_serving/ccsrc/common/instance.h

@@ -0,0 +1,73 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_INSTANCE_H
+#define MINDSPORE_SERVING_INSTANCE_H
+
+#include <vector>
+#include <unordered_map>
+#include <memory>
+#include <string>
+#include <future>
+#include "common/serving_common.h"
+#include "common/servable.h"
+
+namespace mindspore::serving {
+
+using InstanceData = std::vector<TensorBasePtr>;
+using TensorsData = std::vector<TensorBasePtr>;
+struct Instance;
+using WorkerCallBack = std::function<void(const Instance &output, const Status &error_msg)>;
+
+struct WorkerUserContext {
+  WorkerCallBack worker_call_back = nullptr;
+  RequestSpec request_spec;
+  MethodSignature method_def;
+};
+
+struct InstanceContext {
+  uint64_t user_id = 0;
+  uint32_t instance_index = 0;
+
+  WorkerCallBack worker_call_back = nullptr;
+  std::shared_ptr<std::promise<void>> promise = nullptr;
+  std::shared_ptr<WorkerUserContext> user_context = nullptr;
+
+  RequestSpec request_spec;
+  bool operator==(const InstanceContext &other) const {
+    return user_id == other.user_id && instance_index == other.instance_index;
+  }
+};
+
+struct Instance {
+  InstanceData data;  // for inputs of preprocess, predict, postprocess or output
+
+  InstanceData input_data;        // input data
+  InstanceData preprocess_data;   // preprocess result
+  InstanceData predict_data;      // predict result
+  InstanceData postprocess_data;  // postprocess result
+  InstanceContext context;
+  Status error_msg = SUCCESS;
+};
+
+struct ResultInstance {
+  InstanceData data;
+  Status error_msg = SUCCESS;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_INSTANCE_H

mindspore_serving/ccsrc/common/log.cc

@@ -0,0 +1,103 @@
+#include "log.h"
+
+#include <sys/time.h>
+#include "glog/logging.h"
+
+namespace mindspore {
+namespace serving {
+
+#undef Dlog
+#define Dlog(module_id, level, format, ...)                   \
+  do {                                                        \
+    DlogInner((module_id), (level), (format), ##__VA_ARGS__); \
+  } while (0)
+
+static std::string GetTime() {
+#define BUFLEN 80
+  static char buf[BUFLEN];
+#if defined(_WIN32) || defined(_WIN64)
+  time_t time_seconds = time(0);
+  struct tm now_time;
+  localtime_s(&now_time, &time_seconds);
+  sprintf_s(buf, BUFLEN, "%d-%d-%d %d:%d:%d", now_time.tm_year + 1900, now_time.tm_mon + 1, now_time.tm_mday,
+            now_time.tm_hour, now_time.tm_min, now_time.tm_sec);
+#else
+  struct timeval cur_time;
+  (void)gettimeofday(&cur_time, nullptr);
+
+  struct tm now;
+  (void)localtime_r(&cur_time.tv_sec, &now);
+  (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now);  // format date and time
+  // set micro-second
+  buf[27] = '\0';
+  int idx = 26;
+  auto num = cur_time.tv_usec;
+  for (int i = 5; i >= 0; i--) {
+    buf[idx--] = static_cast<char>(num % 10 + '0');
+    num /= 10;
+    if (i % 3 == 0) {
+      buf[idx--] = '.';
+    }
+  }
+#endif
+  return std::string(buf);
+}
+
+static std::string GetProcName() {
+#if defined(__APPLE__) || defined(__FreeBSD__)
+  const char *appname = getprogname();
+#elif defined(_GNU_SOURCE)
+  const char *appname = program_invocation_name;
+#else
+  const char *appname = "?";
+#endif
+  // some times, the appname is an absolute path, its too long
+  std::string app_name(appname);
+  std::size_t pos = app_name.rfind("/");
+  if (pos == std::string::npos) {
+    return app_name;
+  }
+  if (pos + 1 >= app_name.size()) {
+    return app_name;
+  }
+  return app_name.substr(pos + 1);
+}
+
+static std::string GetLogLevel(ERROR_LEVEL level) {
+  switch (level) {
+    case LOG_DEBUG:
+      return "DEBUG";
+    case LOG_INFO:
+      return "INFO";
+    case LOG_WARNING:
+      return "WARNING";
+    case LOG_ERROR:
+    default:
+      return "ERROR";
+  }
+}
+
+// convert MsLogLevel to corresponding glog level
+static int GetGlogLevel(ERROR_LEVEL level) {
+  switch (level) {
+    case LOG_DEBUG:
+    case LOG_INFO:
+      return google::GLOG_INFO;
+    case LOG_WARNING:
+      return google::GLOG_WARNING;
+    case LOG_ERROR:
+    default:
+      return google::GLOG_ERROR;
+  }
+}
+
+void LogWriter::OutputLog(const std::ostringstream &msg) const {
+  auto submodule_name = "Serving";
+  google::LogMessage("", 0, GetGlogLevel(log_level_)).stream()
+    << "[" << GetLogLevel(log_level_) << "] " << submodule_name << "(" << getpid() << "," << GetProcName()
+    << "):" << GetTime() << " "
+    << "[" << file_ << ":" << line_ << "] " << func_ << "] " << msg.str() << std::endl;
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/common/log.h

@@ -0,0 +1,137 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_INC_LOG_H
+#define MINDSPORE_SERVING_INC_LOG_H
+
+#include <iostream>
+#include <vector>
+#include <unordered_map>
+#include <sstream>
+#include <memory>
+#include <string>
+namespace mindspore::serving {
+#define MS_API __attribute__((visibility("default")))
+
+class LogStream {
+ public:
+  LogStream() { sstream_ = std::make_shared<std::stringstream>(); }
+  ~LogStream() = default;
+
+  template <typename T>
+  LogStream &operator<<(const T &val) noexcept {
+    (*sstream_) << val;
+    return *this;
+  }
+
+  template <typename T>
+  LogStream &operator<<(const std::vector<T> &val) noexcept {
+    (*sstream_) << "[";
+    for (size_t i = 0; i < val.size(); i++) {
+      (*this) << val[i];
+      if (i + 1 < val.size()) {
+        (*sstream_) << ", ";
+      }
+    }
+    (*sstream_) << "]";
+    return *this;
+  }
+
+  template <typename K, typename V>
+  LogStream &operator<<(const std::unordered_map<K, V> &val) noexcept {
+    (*sstream_) << "{";
+    for (auto &item : val) {
+      (*this) << "{" << item.first << ": " << item.second << "} ";
+    }
+    (*sstream_) << "}";
+    return *this;
+  }
+
+  LogStream &operator<<(std::ostream &func(std::ostream &os)) noexcept {
+    (*sstream_) << func;
+    return *this;
+  }
+
+  friend class LogWriter;
+  friend class Status;
+
+ private:
+  std::shared_ptr<std::stringstream> sstream_;
+};
+
+enum ERROR_LEVEL {
+  LOG_DEBUG,
+  LOG_INFO,
+  LOG_WARNING,
+  LOG_ERROR,
+  LOG_EXCEPTION,
+};
+
+class MS_API LogWriter {
+ public:
+  LogWriter(const char *file, int line, const char *func, ERROR_LEVEL log_level)
+      : file_(file), line_(line), func_(func), log_level_(log_level) {}
+  ~LogWriter() = default;
+
+  std::string operator<(const LogStream &stream) const noexcept __attribute__((visibility("default"))) {
+    std::ostringstream msg;
+    msg << stream.sstream_->rdbuf();
+    OutputLog(msg);
+    return msg.str();
+  }
+
+  void operator^(const LogStream &stream) const __attribute__((noreturn, visibility("default"))) {
+    std::ostringstream msg;
+    msg << stream.sstream_->rdbuf();
+    OutputLog(msg);
+    throw std::runtime_error(msg.str());
+  }
+
+ private:
+  void OutputLog(const std::ostringstream &msg) const;
+
+  const char *file_;
+  int line_;
+  const char *func_;
+  ERROR_LEVEL log_level_;
+};
+
+#define MSILOG_IF(level)                                                                             \
+  mindspore::serving::LogWriter(__FILE__, __LINE__, __FUNCTION__, mindspore::serving::LOG_##level) < \
+    mindspore::serving::LogStream()
+
+#define MSILOG_THROW                                                                                   \
+  mindspore::serving::LogWriter(__FILE__, __LINE__, __FUNCTION__, mindspore::serving::LOG_EXCEPTION) ^ \
+    mindspore::serving::LogStream()
+
+#define MSI_LOG(level) MSI_LOG_##level
+
+#define MSI_LOG_DEBUG MSILOG_IF(DEBUG)
+#define MSI_LOG_INFO MSILOG_IF(INFO)
+#define MSI_LOG_WARNING MSILOG_IF(WARNING)
+#define MSI_LOG_ERROR MSILOG_IF(ERROR)
+#define MSI_LOG_EXCEPTION MSILOG_THROW
+
+#define MSI_EXCEPTION_IF_NULL(ptr)                                   \
+  do {                                                               \
+    if ((ptr) == nullptr) {                                          \
+      MSI_LOG_EXCEPTION << ": The pointer[" << #ptr << "] is null."; \
+    }                                                                \
+  } while (0)
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_INC_LOG_H

mindspore_serving/ccsrc/common/proto_tensor.cc

@@ -0,0 +1,377 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/proto_tensor.h"
+#include <unordered_map>
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <map>
+#include "common/buffer_tensor.h"
+#include "common/servable.h"
+#include "master/dispacther.h"
+
+using std::string;
+using std::unordered_map;
+using std::vector;
+
+namespace mindspore::serving {
+
+const size_t kMaxShapeElementCount = INT32_MAX;
+const size_t kMaxDataBufferSize = UINT32_MAX;
+
+ProtoTensor::ProtoTensor(proto::Tensor &other) : tensor_(other) {}
+
+ProtoTensor::~ProtoTensor() {}
+
+DataType ProtoTensor::data_type() const { return TransDataType2Inference(tensor_.dtype()); }
+
+void ProtoTensor::set_data_type(DataType data_type) { tensor_.set_dtype(TransDataType2Proto(data_type)); }
+
+std::vector<int64_t> ProtoTensor::shape() const {
+  std::vector<int64_t> result;
+  auto dims = tensor_.shape().dims();
+  std::transform(dims.begin(), dims.end(), std::back_inserter(result), [](const int64_t dim) { return dim; });
+  return result;
+}
+
+void ProtoTensor::set_shape(const std::vector<int64_t> &shape) {
+  auto tensor_shape = tensor_.mutable_shape();
+  tensor_shape->Clear();
+  size_t element_count = 1;
+  for (auto dim : shape) {
+    if (dim <= 0 || element_count > kMaxShapeElementCount / dim) {
+      MSI_LOG_ERROR << "failed to set shape, invalid dim num " << dim;
+      tensor_shape->Clear();
+      return;
+    }
+    element_count *= dim;
+    tensor_shape->add_dims(dim);
+  }
+}
+
+bool ProtoTensor::resize_data(size_t data_len) {
+  string *buffer = tensor_.mutable_data();
+  if (buffer == nullptr) {
+    MSI_LOG_ERROR << "invalid buffer data";
+    return false;
+  }
+  buffer->resize(data_len);
+  return true;
+}
+
+size_t ProtoTensor::data_size() const { return tensor_.data().size(); }
+
+uint8_t *ProtoTensor::mutable_data() {
+  if (data_size() == 0) {
+    return nullptr;
+  }
+  return reinterpret_cast<uint8_t *>(tensor_.mutable_data()->data());
+}
+
+const uint8_t *ProtoTensor::data() const {
+  if (data_size() == 0) {
+    return nullptr;
+  }
+  return reinterpret_cast<const uint8_t *>(tensor_.data().data());
+}
+
+void ProtoTensor::clear_bytes_data() { return tensor_.mutable_bytes_val()->Clear(); }
+
+void ProtoTensor::add_bytes_data(const uint8_t *data, size_t bytes_len) { tensor_.add_bytes_val(data, bytes_len); }
+
+size_t ProtoTensor::bytes_data_size() const { return tensor_.bytes_val().size(); }
+
+void ProtoTensor::get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const {
+  if (index >= static_cast<size_t>(tensor_.bytes_val().size())) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_.bytes_val().size();
+  }
+  auto &bytes = tensor_.bytes_val(index);
+  data = reinterpret_cast<const uint8_t *>(bytes.data());
+  bytes_len = bytes.size();
+}
+
+proto::DataType ProtoTensor::TransDataType2Proto(DataType data_type) {
+  const std::unordered_map<DataType, proto::DataType> id2type_map{
+    {serving::kMSI_Unknown, proto::MS_UNKNOWN}, {serving::kMSI_Bool, proto::MS_BOOL},
+    {serving::kMSI_Float64, proto::MS_FLOAT64}, {serving::kMSI_Int8, proto::MS_INT8},
+    {serving::kMSI_Uint8, proto::MS_UINT8},     {serving::kMSI_Int16, proto::MS_INT16},
+    {serving::kMSI_Uint16, proto::MS_UINT16},   {serving::kMSI_Int32, proto::MS_INT32},
+    {serving::kMSI_Uint32, proto::MS_UINT32},   {serving::kMSI_Int64, proto::MS_INT64},
+    {serving::kMSI_Uint64, proto::MS_UINT64},   {serving::kMSI_Float16, proto::MS_FLOAT16},
+    {serving::kMSI_Float32, proto::MS_FLOAT32}, {serving::kMSI_String, proto::MS_STRING},
+    {serving::kMSI_Bytes, proto::MS_BYTES},
+  };
+  auto it = id2type_map.find(data_type);
+  if (it == id2type_map.end()) {
+    MSI_LOG_WARNING << "failed to set data type, undefined data type " << data_type;
+    return proto::MS_UNKNOWN;
+  } else {
+    return it->second;
+  }
+}
+
+DataType ProtoTensor::TransDataType2Inference(proto::DataType data_type) {
+  const std::unordered_map<proto::DataType, DataType> type2id_map{
+    {proto::MS_UNKNOWN, kMSI_Unknown}, {proto::MS_BOOL, kMSI_Bool},       {proto::MS_INT8, kMSI_Int8},
+    {proto::MS_UINT8, kMSI_Uint8},     {proto::MS_INT16, kMSI_Int16},     {proto::MS_UINT16, kMSI_Uint16},
+    {proto::MS_INT32, kMSI_Int32},     {proto::MS_UINT32, kMSI_Uint32},   {proto::MS_INT64, kMSI_Int64},
+    {proto::MS_UINT64, kMSI_Uint64},   {proto::MS_FLOAT16, kMSI_Float16}, {proto::MS_FLOAT32, kMSI_Float32},
+    {proto::MS_FLOAT64, kMSI_Float64}, {proto::MS_STRING, kMSI_String},   {proto::MS_BYTES, kMSI_Bytes},
+  };
+  auto it = type2id_map.find(data_type);
+  if (it == type2id_map.end()) {
+    MSI_LOG_WARNING << "failed to get data type, undefined data type " << data_type;
+    return kMSI_Unknown;
+  } else {
+    return it->second;
+  }
+}
+
+void GrpcTensorHelper::GetRequestSpec(const proto::PredictRequest &request, RequestSpec &request_spec) {
+  request_spec.servable_name = request.servable_spec().name();
+  request_spec.method_name = request.servable_spec().method_name();
+  request_spec.version_number = request.servable_spec().version_number();
+}
+
+void GrpcTensorHelper::GetWorkerSpec(const proto::RegisterRequest &request, std::vector<WorkerSpec> &worker_specs) {
+  for (auto &proto_spec : request.worker_spec()) {
+    WorkerSpec worker_spec;
+    worker_spec.worker_address = request.address();
+    worker_spec.servable_name = proto_spec.name();
+    worker_spec.version_number = proto_spec.version_number();
+    for (const auto &proto_method : proto_spec.methods()) {
+      WorkerMethodInfo method_info;
+      method_info.name = proto_method.name();
+      for (auto &name : proto_method.input_names()) {
+        method_info.input_names.push_back(name);
+      }
+      worker_spec.methods.push_back(method_info);
+    }
+    worker_specs.push_back(worker_spec);
+  }
+}
+
+void GrpcTensorHelper::GetWorkerSpec(const proto::AddWorkerRequest &request, WorkerSpec &worker_spec) {
+  worker_spec.worker_address = request.address();
+  worker_spec.servable_name = request.worker_spec().name();
+  worker_spec.version_number = request.worker_spec().version_number();
+  for (const auto &proto_method : request.worker_spec().methods()) {
+    WorkerMethodInfo method_info;
+    method_info.name = proto_method.name();
+    for (auto &name : proto_method.input_names()) {
+      method_info.input_names.push_back(name);
+    }
+    worker_spec.methods.push_back(method_info);
+  }
+}
+
+void GrpcTensorHelper::GetWorkerSpec(const proto::RemoveWorkerRequest &request, WorkerSpec &worker_spec) {
+  worker_spec.worker_address = request.address();
+  worker_spec.servable_name = request.worker_spec().name();
+  worker_spec.version_number = request.worker_spec().version_number();
+  for (const auto &proto_method : request.worker_spec().methods()) {
+    WorkerMethodInfo method_info;
+    method_info.name = proto_method.name();
+    for (auto &name : proto_method.input_names()) {
+      method_info.input_names.push_back(name);
+    }
+    worker_spec.methods.push_back(method_info);
+  }
+}
+
+Status GrpcTensorHelper::CreateInstanceFromRequest(const proto::PredictRequest &request, RequestSpec &request_spec,
+                                                   vector<InstanceData> &results) {
+  results.clear();
+
+  Status status;
+  GetRequestSpec(request, request_spec);
+
+  auto servable_name = request_spec.servable_name;
+  auto method_name = request_spec.method_name;
+
+  ServableSignature servable_signature;
+  if (!ServableStorage::Instance()->GetServableDef(servable_name, servable_signature)) {
+    return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Servable " << servable_name << " is not declared";
+  }
+  MethodSignature method_signature;
+  if (!servable_signature.GetMethodDeclare(request_spec.method_name, method_signature)) {
+    return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+           << "Method " << method_name << " is not registed for servable " << servable_name;
+  }
+
+  // instance
+  if (request.instances_size() > 0) {
+    status = CreateInstanceFromRequestInstances(request, method_signature.inputs, results);
+    if (status != SUCCESS) {
+      MSI_LOG_ERROR << "Create instances from request instances failed";
+      return status;
+    }
+  }
+  return SUCCESS;
+}
+
+Status GrpcTensorHelper::CreateReplyFromInstances(const proto::PredictRequest &request, const vector<Instance> &outputs,
+                                                  proto::PredictReply &reply) {
+  auto servable_name = request.servable_spec().name();
+  auto method_name = request.servable_spec().method_name();
+  Status status;
+  ServableSignature servable_signature;
+  if (!ServableStorage::Instance()->GetServableDef(servable_name, servable_signature)) {
+    return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Servable " << servable_name << " is not declared";
+  }
+  MethodSignature method_signature;
+  if (!servable_signature.GetMethodDeclare(method_name, method_signature)) {
+    return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+           << "Method " << method_name << " is not registed for servable " << servable_name;
+  }
+  *reply.mutable_servable_spec() = request.servable_spec();
+
+  size_t err_cnt = 0;
+  for (auto &output_intance : outputs) {
+    if (output_intance.error_msg != SUCCESS) {
+      err_cnt++;
+    } else if (output_intance.data.size() != method_signature.outputs.size()) {
+      return INFER_STATUS_LOG_ERROR(SYSTEM_ERROR)
+             << "Result data tensor size " << output_intance.data.size() << " not equal outputs size "
+             << method_signature.outputs.size() << "defined in method signature";
+    }
+  }
+  if (err_cnt > 0) {
+    for (auto &output_intance : outputs) {
+      auto proto_err_msg = reply.add_error_msg();
+      proto_err_msg->set_error_code(output_intance.error_msg.StatusCode());
+      if (output_intance.error_msg == INVALID_INPUTS) {
+        proto_err_msg->set_error_msg(output_intance.error_msg.StatusMessage());
+      } else if (output_intance.error_msg != SUCCESS) {
+        proto_err_msg->set_error_msg(output_intance.error_msg.StatusMessage());
+      }
+    }
+  }
+  // create instance reply, same with request
+  if (request.instances_size() > 0) {
+    for (auto &output_intance : outputs) {
+      auto proto_instance = reply.add_instances();
+      if (output_intance.data.empty()) {
+        continue;
+      }
+      auto proto_items = proto_instance->mutable_items();
+      for (size_t i = 0; i < method_signature.outputs.size(); i++) {
+        auto &output_tensor = output_intance.data[i];
+        ProtoTensor proto_tensor((*proto_items)[method_signature.outputs[i]]);
+        proto_tensor.assgin(*output_tensor);
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status GrpcTensorHelper::CreateInstanceFromRequestInstances(const proto::PredictRequest &request,
+                                                            const std::vector<std::string> &input_names,
+                                                            std::vector<InstanceData> &results) {
+  auto servable_name = request.servable_spec().name();
+  auto method_name = request.servable_spec().method_name();
+  Status status;
+  for (auto &proto_instance : request.instances()) {
+    InstanceData instance_data;
+    for (const auto &input_name : input_names) {
+      auto it = proto_instance.items().find(input_name);
+      if (it == proto_instance.items().end()) {
+        return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+               << "Cannot find input " << input_name << " in instance input , servable " << servable_name << ", method "
+               << method_name;
+      }
+      status = CheckRequestTensor(it->second, true, 1);
+      if (status != SUCCESS) {
+        auto status2 = INFER_STATUS(INVALID_INPUTS) << "Instances input " << input_name << " check failed";
+        MSI_LOG_ERROR << status2.StatusMessage();
+        return Status(INVALID_INPUTS, status2.StatusMessage() + ", detail: " + status.StatusMessage());
+      }
+      auto add_tensor = std::make_shared<ProtoTensor>(const_cast<proto::Tensor &>(it->second));
+      instance_data.push_back(add_tensor);
+    }
+    results.push_back(instance_data);
+  }
+  return SUCCESS;
+}
+
+Status GrpcTensorHelper::CheckRequestTensor(const proto::Tensor &tensor, bool is_instance_tensor, uint32_t batch_size) {
+  Status status;
+  ProtoTensor tensor_input(const_cast<proto::Tensor &>(tensor));
+  auto shape = tensor_input.shape();
+  if (tensor.dtype() == proto::MS_BYTES || tensor.dtype() == proto::MS_STRING) {
+    if (is_instance_tensor) {
+      if (tensor.bytes_val_size() != 1) {
+        return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+               << "Instance tensor check failed: bytes or string type shape batch size " << batch_size
+               << " not equal to bytes val size " << tensor.bytes_val_size();
+      }
+      if (!(shape.size() == 1 && shape[0] == 1) && !shape.empty()) {
+        return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+               << "Instance tensor check failed: bytes or string type input "
+               << " shape can only be (1,) or empty, but given shape is " << shape;
+      }
+    } else {
+      if (static_cast<int64_t>(tensor.bytes_val_size()) != batch_size) {
+        return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+               << "Inputs tensor check failed: bytes or string type shape batch size " << batch_size
+               << " not equal to bytes val size " << tensor.bytes_val_size();
+      }
+      if (shape.size() != 1) {
+        return INFER_STATUS_LOG_ERROR(INVALID_INPUTS)
+               << "Inputs Tensor check failed: bytes or string type input "
+               << " shape can only be (batch_size,), but given shape is " << shape;
+      }
+    }
+  } else {
+    size_t element_num = tensor_input.element_cnt();
+    if (element_num == 0) {  // shape dim invalid or element count > max element num
+      return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Tensor check failed: input "
+                                                    << " shape " << tensor_input.shape() << " invalid";
+    }
+    if (tensor_input.data_type() == kMSI_Unknown || tensor_input.itemsize() == 0) {
+      return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Tensor check failed: input "
+                                                    << " data type " << tensor.dtype() << " invalid";
+    }
+    if (element_num * tensor_input.itemsize() != tensor_input.data_size()) {
+      return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Tensor check failed: input "
+                                                    << " data size " << tensor.data().size() << " invalid";
+    }
+  }
+  return SUCCESS;
+}
+
+serving::LogStream &operator<<(serving::LogStream &stream, proto::DataType data_type) {
+  const std::map<proto::DataType, std::string> type_name_map{
+    {proto::MS_UNKNOWN, "proto::MS_UNKNOWN"}, {proto::MS_BOOL, "proto::kMSI_Bool"},
+    {proto::MS_INT8, "proto::MS_INT8"},       {proto::MS_UINT8, "proto::MS_UINT8"},
+    {proto::MS_INT16, "proto::MS_INT16"},     {proto::MS_UINT16, "proto::MS_UINT16"},
+    {proto::MS_INT32, "proto::MS_INT32"},     {proto::MS_UINT32, "proto::MS_UINT32"},
+    {proto::MS_INT64, "proto::MS_INT64"},     {proto::MS_UINT64, "proto::MS_UINT64"},
+    {proto::MS_FLOAT16, "proto::MS_FLOAT16"}, {proto::MS_FLOAT32, "proto::MS_FLOAT32"},
+    {proto::MS_FLOAT64, "proto::MS_FLOAT64"}, {proto::MS_STRING, "proto::MS_STRING"},
+    {proto::MS_BYTES, "proto::MS_BYTES"},
+  };
+  auto it = type_name_map.find(data_type);
+  if (it != type_name_map.end()) {
+    stream << it->second;
+  } else {
+    stream << "proto::MS_UNKNOWN";
+  }
+  return stream;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/proto_tensor.h

@@ -0,0 +1,82 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_TENSOR_H_
+#define MINDSPORE_SERVING_TENSOR_H_
+
+#include <string>
+#include <utility>
+#include <vector>
+#include <memory>
+#include "common/serving_common.h"
+#include "proto/ms_service.pb.h"
+#include "proto/ms_master.pb.h"
+#include "common/instance.h"
+#include "common/servable.h"
+
+namespace mindspore::serving {
+
+class MS_API ProtoTensor : public TensorBase {
+ public:
+  // the other's lifetime must longer than this object
+  explicit ProtoTensor(proto::Tensor &other);
+  ~ProtoTensor();
+
+  DataType data_type() const override;
+  void set_data_type(DataType type) override;
+  std::vector<int64_t> shape() const override;
+  void set_shape(const std::vector<int64_t> &shape) override;
+  const uint8_t *data() const override;
+  size_t data_size() const override;
+  bool resize_data(size_t data_len) override;
+  uint8_t *mutable_data() override;
+
+  void clear_bytes_data() override;
+  void add_bytes_data(const uint8_t *data, size_t bytes_len) override;
+  size_t bytes_data_size() const override;
+  void get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const override;
+
+  static proto::DataType TransDataType2Proto(DataType data_type);
+  static DataType TransDataType2Inference(proto::DataType data_type);
+
+ private:
+  // if tensor_ is reference from other ms_serving::Tensor, the other's lifetime must
+  // longer than this object
+  proto::Tensor &tensor_;
+};
+
+class MS_API GrpcTensorHelper {
+ public:
+  static void GetRequestSpec(const proto::PredictRequest &request, RequestSpec &request_spec);
+  static void GetWorkerSpec(const proto::RegisterRequest &request, std::vector<WorkerSpec> &worker_specs);
+  static void GetWorkerSpec(const proto::AddWorkerRequest &request, WorkerSpec &worker_spec);
+  static void GetWorkerSpec(const proto::RemoveWorkerRequest &request, WorkerSpec &worker_spec);
+  static Status CreateInstanceFromRequest(const proto::PredictRequest &request, RequestSpec &request_spec,
+                                          std::vector<InstanceData> &results);
+  static Status CreateReplyFromInstances(const proto::PredictRequest &request, const std::vector<Instance> &inputs,
+                                         proto::PredictReply &reply);
+
+ private:
+  static Status CreateInstanceFromRequestInstances(const proto::PredictRequest &request,
+                                                   const std::vector<std::string> &input_names,
+                                                   std::vector<InstanceData> &results);
+  static Status CheckRequestTensor(const proto::Tensor &tensor, bool is_instance_tensor, uint32_t batch_size);
+};
+
+extern MS_API LogStream &operator<<(serving::LogStream &stream, proto::DataType data_type);
+
+}  // namespace mindspore::serving
+#endif  // MINDSPORE_SERVING_TENSOR_H_

mindspore_serving/ccsrc/common/servable.cc

@@ -0,0 +1,295 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/servable.h"
+#include <set>
+#include <sstream>
+#include "worker/preprocess.h"
+#include "worker/postprocess.h"
+
+namespace mindspore::serving {
+
+std::string ServableMeta::Repr() const {
+  std::ostringstream stream;
+  stream << "path(" << servable_name << ") file(" << servable_file + ")";
+  return stream.str();
+}
+
+void ServableMeta::SetModelFormat(const std::string &format) {
+  if (format == "om") {
+    model_format = kOM;
+  } else if (format == "mindir") {
+    model_format = kMindIR;
+  } else {
+    MSI_LOG_ERROR << "Invalid model format " << format;
+  }
+}
+
+std::string LoadServableSpec::Repr() const {
+  std::string version;
+  if (version_number > 0) {
+    version = " version(" + std::to_string(version_number) + ") ";
+  }
+  return "servable(" + servable_name + ") " + version;
+}
+
+std::string WorkerSpec::Repr() const {
+  std::string version;
+  if (version_number > 0) {
+    version = " version(" + std::to_string(version_number) + ") ";
+  }
+  return "servable(" + servable_name + ") " + version + " address(" + worker_address + ") ";
+}
+
+std::string RequestSpec::Repr() const {
+  std::string version;
+  if (version_number > 0) {
+    version = " version(" + std::to_string(version_number) + ") ";
+  }
+  return "servable(" + servable_name + ") " + "method(" + method_name + ") " + version;
+}
+
+Status ServableSignature::Check() const {
+  std::set<std::string> method_set;
+  std::string model_str = servable_meta.Repr();
+
+  for (auto &method : methods) {
+    if (method_set.count(method.method_name) > 0) {
+      return INFER_STATUS_LOG_ERROR(FAILED)
+             << "Model " << model_str << " " << method.method_name << " has been defined repeatly";
+    }
+    method_set.emplace(method.method_name);
+
+    size_t preprocess_outputs_count = 0;
+    size_t postprocess_outputs_count = 0;
+
+    const auto &preprocess_name = method.preprocess_name;
+    if (!preprocess_name.empty()) {
+      auto preprocess = PreprocessStorage::Instance().GetPreprocess(preprocess_name);
+      if (preprocess == nullptr) {
+        return INFER_STATUS_LOG_ERROR(FAILED) << "Model " << model_str << " method " << method.method_name
+                                              << " preprocess " << preprocess_name << " not defined";
+      }
+      preprocess_outputs_count = preprocess->GetOutputsCount(preprocess_name);
+
+      for (size_t i = 0; i < method.preprocess_inputs.size(); i++) {
+        auto &input = method.preprocess_inputs[i];
+        if (input.first != kPredictPhaseTag_Input) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the preprocess " << i
+                 << "th input phase tag " << input.first << " invalid";
+        }
+        if (input.second >= method.inputs.size()) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the preprocess " << i
+                 << "th input uses method " << input.second << "th input, that is greater than the method inputs size "
+                 << method.inputs.size();
+        }
+      }
+    }
+
+    for (size_t i = 0; i < method.servable_inputs.size(); i++) {
+      auto &input = method.servable_inputs[i];
+      if (input.first == kPredictPhaseTag_Input) {
+        if (input.second >= method.inputs.size()) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the servable " << i
+                 << "th input uses method " << input.second << "th input, that is greater than the method inputs size "
+                 << method.inputs.size();
+        }
+      } else if (input.first == kPredictPhaseTag_Preproces) {
+        if (input.second >= preprocess_outputs_count) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the servable " << i
+                 << "th input uses preprocess " << input.second
+                 << "th output, that is greater than the preprocess outputs size " << preprocess_outputs_count;
+        }
+      } else {
+        return INFER_STATUS_LOG_ERROR(FAILED)
+               << "Model " << model_str << " method " << method.method_name << ", the servable " << i
+               << "th input phase tag " << input.first << " invalid";
+      }
+    }
+
+    const auto &postprocess_name = method.postprocess_name;
+    if (!method.postprocess_name.empty()) {
+      auto postprocess = PostprocessStorage::Instance().GetPostprocess(postprocess_name);
+      if (postprocess == nullptr) {
+        return INFER_STATUS_LOG_ERROR(FAILED) << "Model " << model_str << " method " << method.method_name
+                                              << " postprocess " << postprocess_name << " not defined";
+      }
+      postprocess_outputs_count = postprocess->GetOutputsCount(postprocess_name);
+
+      for (size_t i = 0; i < method.postprocess_inputs.size(); i++) {
+        auto &input = method.postprocess_inputs[i];
+        if (input.first == kPredictPhaseTag_Input) {
+          if (input.second >= method.inputs.size()) {
+            return INFER_STATUS_LOG_ERROR(FAILED)
+                   << "Model " << model_str << " method " << method.method_name << ", the postprocess " << i
+                   << "th input uses method " << input.second
+                   << "th input, that is greater than the method inputs size " << method.inputs.size();
+          }
+        } else if (input.first == kPredictPhaseTag_Preproces) {
+          if (input.second >= preprocess_outputs_count) {
+            return INFER_STATUS_LOG_ERROR(FAILED)
+                   << "Model " << model_str << " method " << method.method_name << ", the postprocess " << i
+                   << "th input uses preprocess " << input.second
+                   << "th output, that is greater than the preprocess outputs size " << preprocess_outputs_count;
+          }
+        } else if (input.first == kPredictPhaseTag_Predict) {
+          if (input.second >= servable_meta.outputs_count) {
+            return INFER_STATUS_LOG_ERROR(FAILED)
+                   << "Model " << model_str << " method " << method.method_name << ", the postprocess " << i
+                   << "th input uses servable " << input.second
+                   << "th output, that is greater than the servable outputs size " << servable_meta.outputs_count;
+          }
+        } else {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the postprocess " << i
+                 << "th input phase tag " << input.first << " invalid";
+        }
+      }
+    }
+    for (size_t i = 0; i < method.returns.size(); i++) {
+      auto &input = method.returns[i];
+      if (input.first == kPredictPhaseTag_Input) {
+        if (input.second >= method.inputs.size()) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the method " << i
+                 << "th output uses method " << input.second << "th input, that is greater than the method inputs size "
+                 << method.inputs.size();
+        }
+      } else if (input.first == kPredictPhaseTag_Preproces) {
+        if (input.second >= preprocess_outputs_count) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the method " << i
+                 << "th output uses preprocess " << input.second
+                 << "th output, that is greater than the preprocess outputs size " << preprocess_outputs_count;
+        }
+      } else if (input.first == kPredictPhaseTag_Predict) {
+        if (input.second >= servable_meta.outputs_count) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the method " << i
+                 << "th output uses servable " << input.second
+                 << "th output, that is greater than the servable outputs size " << servable_meta.outputs_count;
+        }
+      } else if (input.first == kPredictPhaseTag_Postprocess) {
+        if (input.second >= postprocess_outputs_count) {
+          return INFER_STATUS_LOG_ERROR(FAILED)
+                 << "Model " << model_str << " method " << method.method_name << ", the method " << i
+                 << "th output uses postprocess " << input.second
+                 << "th output, that is greater than the postprocess outputs size " << postprocess_outputs_count;
+        }
+      } else {
+        return INFER_STATUS_LOG_ERROR(FAILED)
+               << "Model " << model_str << " method " << method.method_name << ", the method " << i
+               << "th output phase tag " << input.first << " invalid";
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+bool ServableSignature::GetMethodDeclare(const std::string &method_name, MethodSignature &method) {
+  auto item =
+    find_if(methods.begin(), methods.end(), [&](const MethodSignature &v) { return v.method_name == method_name; });
+  if (item != methods.end()) {
+    method = *item;
+    return true;
+  }
+  return false;
+}
+
+void ServableStorage::Register(const ServableSignature &def) {
+  auto model_name = def.servable_meta.servable_name;
+  if (servable_signatures_map_.find(model_name) == servable_signatures_map_.end()) {
+    MSI_LOG_WARNING << "Servable " << model_name << " has already been defined";
+  }
+  servable_signatures_map_[model_name] = def;
+}
+
+bool ServableStorage::GetServableDef(const std::string &model_name, ServableSignature &def) const {
+  auto it = servable_signatures_map_.find(model_name);
+  if (it == servable_signatures_map_.end()) {
+    return false;
+  }
+  def = it->second;
+  return true;
+}
+
+std::shared_ptr<ServableStorage> ServableStorage::Instance() {
+  static std::shared_ptr<ServableStorage> storage;
+  if (storage == nullptr) {
+    storage = std::make_shared<ServableStorage>();
+  }
+  return storage;
+}
+
+void ServableStorage::RegisterMethod(const MethodSignature &method) {
+  MSI_LOG_INFO << "Declare method " << method.method_name << ", servable " << method.servable_name;
+  auto it = servable_signatures_map_.find(method.servable_name);
+  if (it == servable_signatures_map_.end()) {
+    ServableSignature signature;
+    signature.methods.push_back(method);
+    servable_signatures_map_[method.servable_name] = signature;
+    return;
+  }
+  it->second.methods.push_back(method);
+}
+
+void ServableStorage::DeclareServable(const mindspore::serving::ServableMeta &servable) {
+  MSI_LOG_INFO << "Declare servable " << servable.servable_name;
+  auto it = servable_signatures_map_.find(servable.servable_name);
+  if (it == servable_signatures_map_.end()) {
+    ServableSignature signature;
+    signature.servable_meta = servable;
+    servable_signatures_map_[servable.servable_name] = signature;
+    return;
+  }
+  it->second.servable_meta = servable;
+}
+
+void ServableStorage::RegisterInputOutputInfo(const std::string &servable_name, size_t inputs_count,
+                                              size_t outputs_count) {
+  auto it = servable_signatures_map_.find(servable_name);
+  if (it == servable_signatures_map_.end()) {
+    MSI_LOG_EXCEPTION << "RegisterInputOutputInfo failed, cannot find servable " << servable_name;
+  }
+  auto &servable_meta = it->second.servable_meta;
+  if (servable_meta.inputs_count != 0 && servable_meta.inputs_count != inputs_count) {
+    MSI_LOG_EXCEPTION << "RegisterInputOutputInfo failed, inputs count " << inputs_count << " not match old count "
+                      << servable_meta.inputs_count << ",servable name " << servable_name;
+  }
+  if (servable_meta.outputs_count != 0 && servable_meta.outputs_count != outputs_count) {
+    MSI_LOG_EXCEPTION << "RegisterInputOutputInfo failed, outputs count " << outputs_count << " not match old count "
+                      << servable_meta.outputs_count << ",servable name " << servable_name;
+  }
+  servable_meta.inputs_count = inputs_count;
+  servable_meta.outputs_count = outputs_count;
+}
+
+std::vector<size_t> ServableStorage::GetInputOutputInfo(const std::string &servable_name) const {
+  std::vector<size_t> result;
+  auto it = servable_signatures_map_.find(servable_name);
+  if (it == servable_signatures_map_.end()) {
+    return result;
+  }
+  result.push_back(it->second.servable_meta.inputs_count);
+  result.push_back(it->second.servable_meta.outputs_count);
+  return result;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/servable.h

@@ -0,0 +1,122 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_SERVABLE_H
+#define MINDSPORE_SERVING_SERVABLE_H
+
+#include <utility>
+#include <unordered_map>
+#include <string>
+#include <vector>
+#include <memory>
+#include "common/serving_common.h"
+#include "worker/inference/inference.h"
+
+namespace mindspore::serving {
+
+enum PredictPhaseTag {
+  kPredictPhaseTag_Input,
+  kPredictPhaseTag_Preproces,
+  kPredictPhaseTag_Predict,
+  kPredictPhaseTag_Postprocess,
+  kPredictPhaseTag_Output,
+};
+
+struct MethodSignature {
+  std::string method_name;
+  std::vector<std::string> inputs;
+  std::vector<std::string> outputs;
+
+  std::string preprocess_name;
+  // the output index of the 'predict phase'
+  std::vector<std::pair<PredictPhaseTag, uint64_t>> preprocess_inputs;
+
+  std::string postprocess_name;
+  std::vector<std::pair<PredictPhaseTag, uint64_t>> postprocess_inputs;
+
+  std::string servable_name;
+  std::vector<std::pair<PredictPhaseTag, uint64_t>> servable_inputs;
+
+  std::vector<std::pair<PredictPhaseTag, uint64_t>> returns;
+};
+
+struct LoadServableSpec {
+  std::string servable_directory;
+  std::string servable_name;
+  uint64_t version_number = 0;
+  std::string Repr() const;
+};
+
+struct WorkerMethodInfo {
+  std::string name;
+  std::vector<std::string> input_names;
+};
+
+struct WorkerSpec {
+  std::string servable_name;
+  uint64_t version_number = 0;
+  std::string worker_address;
+  std::vector<WorkerMethodInfo> methods;
+  std::string Repr() const;
+};
+
+struct RequestSpec {
+  std::string servable_name;
+  std::string method_name;
+  uint64_t version_number = 0;  // not specified
+  std::string Repr() const;
+};
+
+struct MS_API ServableMeta {
+  std::string servable_name;
+  std::string servable_file;   // file name
+  ModelType model_format;      // OM， MindIR
+  bool with_batch_dim = true;  // whether there is batch dim in model's inputs/outputs
+  size_t inputs_count = 0;
+  size_t outputs_count = 0;
+  std::string Repr() const;
+  void SetModelFormat(const std::string &format);
+};
+
+struct ServableSignature {
+  ServableMeta servable_meta;
+  std::vector<MethodSignature> methods;
+
+  Status Check() const;
+  bool GetMethodDeclare(const std::string &method_name, MethodSignature &method);
+};
+
+class MS_API ServableStorage {
+ public:
+  void Register(const ServableSignature &def);
+  void RegisterMethod(const MethodSignature &method);
+
+  bool GetServableDef(const std::string &model_name, ServableSignature &def) const;
+
+  void DeclareServable(const ServableMeta &servable);
+
+  void RegisterInputOutputInfo(const std::string &servable_name, size_t inputs_count, size_t outputs_count);
+  std::vector<size_t> GetInputOutputInfo(const std::string &servable_name) const;
+
+  static std::shared_ptr<ServableStorage> Instance();
+
+ private:
+  std::unordered_map<std::string, ServableSignature> servable_signatures_map_;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_SERVABLE_H

mindspore_serving/ccsrc/common/serving_common.h

@@ -0,0 +1,26 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_COMMON_H
+#define MINDSPORE_SERVING_COMMON_H
+
+#include <securec.h>
+
+#include "common/status.h"
+#include "common/log.h"
+#include "common/tensor.h"
+
+#endif  // MINDSPORE_SERVING_COMMON_H

mindspore_serving/ccsrc/common/status.h

@@ -0,0 +1,75 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_INC_STATUS_H
+#define MINDSPORE_SERVING_INC_STATUS_H
+
+#include <chrono>
+#include <string>
+#include <sstream>
+
+#include "log.h"
+
+namespace mindspore::serving {
+
+enum StatusCode { SUCCESS = 0, FAILED, INVALID_INPUTS, SYSTEM_ERROR };
+
+class Status {
+ public:
+  Status() : status_code_(FAILED) {}
+  Status(enum StatusCode status_code, const std::string &status_msg = "")
+      : status_code_(status_code), status_msg_(status_msg) {}
+  bool IsSuccess() const { return status_code_ == SUCCESS; }
+  enum StatusCode StatusCode() const { return status_code_; }
+  std::string StatusMessage() const { return status_msg_; }
+  bool operator==(const Status &other) const { return status_code_ == other.status_code_; }
+  bool operator==(enum StatusCode other_code) const { return status_code_ == other_code; }
+  bool operator!=(const Status &other) const { return status_code_ != other.status_code_; }
+  bool operator!=(enum StatusCode other_code) const { return status_code_ != other_code; }
+  operator bool() const = delete;
+  Status &operator<(const LogStream &stream) noexcept __attribute__((visibility("default"))) {
+    status_msg_ = stream.sstream_->str();
+    return *this;
+  }
+  Status &operator=(const std::string &msg) noexcept __attribute__((visibility("default"))) {
+    status_msg_ = msg;
+    return *this;
+  }
+
+ private:
+  enum StatusCode status_code_;
+  std::string status_msg_;
+};
+
+#define MSI_TIME_STAMP_START(name) auto time_start_##name = std::chrono::steady_clock::now();
+#define MSI_TIME_STAMP_END(name)                                                                             \
+  {                                                                                                          \
+    auto time_end_##name = std::chrono::steady_clock::now();                                                 \
+    auto time_cost = std::chrono::duration<double, std::milli>(time_end_##name - time_start_##name).count(); \
+    MSI_LOG_INFO << #name " Time Cost # " << time_cost << " ms ---------------------";                       \
+  }
+
+#define INFER_STATUS(code) mindspore::serving::Status(code) < mindspore::serving::LogStream()
+#define ERROR_INFER_STATUS(status, type, msg) \
+  MSI_LOG_ERROR << msg;                       \
+  status = mindspore::serving::Status(type, msg)
+
+#define INFER_STATUS_LOG_ERROR(code) mindspore::serving::Status(code) = MSI_LOG_ERROR
+#define INFER_STATUS_LOG_WARNING(code) mindspore::serving::Status(code) = MSI_LOG_WARNING
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_INC_STATUS_H

mindspore_serving/ccsrc/common/tensor.cc

@@ -0,0 +1,113 @@
+#include "tensor.h"
+#include <functional>
+#include "log.h"
+#include "securec.h"
+
+namespace mindspore::serving {
+
+Tensor::Tensor() = default;
+
+Tensor::Tensor(DataType type, std::vector<int64_t> shape, const void *data, size_t data_len) {
+  set_data_type(type);
+  set_shape(shape);
+  set_data(data, data_len);
+}
+
+const uint8_t *Tensor::data() const {
+  if (data_size() == 0) {
+    return nullptr;
+  }
+  return data_.data();
+}
+
+size_t Tensor::data_size() const { return data_.size(); }
+
+bool Tensor::resize_data(size_t data_len) {
+  data_.resize(data_len);
+  return true;
+}
+
+uint8_t *Tensor::mutable_data() {
+  if (data_size() == 0) {
+    return nullptr;
+  }
+  return data_.data();
+}
+
+// For kMSI_String and kMSI_Bytes
+void Tensor::clear_bytes_data() { bytes_.clear(); }
+
+void Tensor::add_bytes_data(const uint8_t *data, size_t bytes_len) {
+  std::vector<uint8_t> bytes(bytes_len);
+  memcpy_s(bytes.data(), bytes.size(), data, bytes_len);
+  bytes_.push_back(std::move(bytes));
+}
+
+size_t Tensor::bytes_data_size() const { return bytes_.size(); }
+
+void Tensor::get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const {
+  bytes_len = bytes_[index].size();
+  if (bytes_len == 0) {
+    data = nullptr;
+  } else {
+    data = bytes_[index].data();
+  }
+}
+
+TensorBase *VectorTensorWrapReply::operator[](size_t index) {
+  if (index >= tensor_list_.size()) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_list_.size();
+  }
+  return &(tensor_list_[index]);
+}
+
+const TensorBase *VectorTensorWrapReply::operator[](size_t index) const {
+  if (index >= tensor_list_.size()) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_list_.size();
+  }
+  return &(tensor_list_[index]);
+}
+
+TensorBase *VectorTensorWrapReply::add() {
+  tensor_list_.push_back(Tensor());
+  return &(tensor_list_.back());
+}
+
+const TensorBase *VectorTensorWrapRequest::operator[](size_t index) const {
+  if (index >= tensor_list_.size()) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_list_.size();
+  }
+  return &(tensor_list_[index]);
+}
+
+TensorBase *VectorTensorPtrWrapReply::operator[](size_t index) {
+  if (index >= tensor_list_.size()) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_list_.size();
+  }
+  return tensor_list_[index].get();
+}
+
+const TensorBase *VectorTensorPtrWrapReply::operator[](size_t index) const {
+  if (index >= tensor_list_.size()) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_list_.size();
+  }
+  return tensor_list_[index].get();
+}
+
+TensorBase *VectorTensorPtrWrapReply::add() {
+  auto tensor = tensor_create_fun_();
+  if (tensor == nullptr) {
+    MSI_LOG_EXCEPTION << "create tensor failed";
+  }
+  tensor_list_.push_back(tensor);
+  return tensor.get();
+}
+
+const TensorBase *VectorTensorPtrWrapRequest::operator[](size_t index) const {
+  if (index >= tensor_list_.size()) {
+    MSI_LOG_EXCEPTION << "visit invalid index " << index << " total size " << tensor_list_.size();
+  }
+  return tensor_list_[index].get();
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/tensor.h

@@ -0,0 +1,119 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_INC_TENSOR_IMP_H
+#define MINDSPORE_SERVING_INC_TENSOR_IMP_H
+
+#include <vector>
+#include "tensor_base.h"
+
+namespace mindspore::serving {
+
+class MS_API Tensor : public TensorBase {
+ public:
+  Tensor();
+  Tensor(DataType type, std::vector<int64_t> shape, const void *data, size_t data_len);
+  ~Tensor() = default;
+
+  void set_data_type(DataType type) override { type_ = type; }
+  DataType data_type() const override { return type_; }
+
+  void set_shape(const std::vector<int64_t> &shape) override { shape_ = shape; }
+  std::vector<int64_t> shape() const override { return shape_; }
+
+  const uint8_t *data() const override;
+  size_t data_size() const override;
+
+  bool resize_data(size_t data_len) override;
+  uint8_t *mutable_data() override;
+
+  // For kMSI_String and kMSI_Bytes
+  void clear_bytes_data() override;
+  void add_bytes_data(const uint8_t *data, size_t bytes_len) override;
+  size_t bytes_data_size() const override;
+  void get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const override;
+
+ private:
+  DataType type_;
+  std::vector<int64_t> shape_;
+  std::vector<uint8_t> data_;
+  // For kMSI_String and kMSI_Bytes
+  std::vector<std::vector<uint8_t>> bytes_;
+};
+
+class MS_API VectorTensorWrapReply : public ReplyBase {
+ public:
+  explicit VectorTensorWrapReply(std::vector<Tensor> &tensor_list) : tensor_list_(tensor_list) {}
+  ~VectorTensorWrapReply() = default;
+
+  size_t size() const override { return tensor_list_.size(); }
+  TensorBase *operator[](size_t index) override;
+  const TensorBase *operator[](size_t index) const override;
+  TensorBase *add() override;
+  void clear() override { tensor_list_.clear(); }
+
+ private:
+  std::vector<Tensor> &tensor_list_;
+};
+
+class MS_API VectorTensorWrapRequest : public RequestBase {
+ public:
+  explicit VectorTensorWrapRequest(const std::vector<Tensor> &tensor_list) : tensor_list_(tensor_list) {}
+  ~VectorTensorWrapRequest() = default;
+
+  size_t size() const override { return tensor_list_.size(); }
+  const TensorBase *operator[](size_t index) const override;
+
+ private:
+  const std::vector<Tensor> &tensor_list_;
+};
+
+class MS_API VectorTensorPtrWrapReply : public ReplyBase {
+ public:
+  explicit VectorTensorPtrWrapReply(std::vector<TensorBasePtr> &tensor_list, std::function<TensorBasePtr()> create_fun)
+      : tensor_list_(tensor_list), tensor_create_fun_(create_fun) {
+    if (tensor_create_fun_ == nullptr) {
+      MSI_LOG_EXCEPTION << "tensor create function cannot be nullptr";
+    }
+  }
+  ~VectorTensorPtrWrapReply() = default;
+
+  size_t size() const override { return tensor_list_.size(); }
+  TensorBase *operator[](size_t index) override;
+  const TensorBase *operator[](size_t index) const override;
+  TensorBase *add() override;
+  void clear() override { tensor_list_.clear(); }
+
+ private:
+  std::vector<TensorBasePtr> &tensor_list_;
+  std::function<TensorBasePtr()> tensor_create_fun_;
+};
+
+class MS_API VectorTensorPtrWrapRequest : public RequestBase {
+ public:
+  explicit VectorTensorPtrWrapRequest(const std::vector<TensorBasePtr> &tensor_list) : tensor_list_(tensor_list) {}
+  ~VectorTensorPtrWrapRequest() = default;
+
+  size_t size() const override { return tensor_list_.size(); }
+  const TensorBase *operator[](size_t index) const override;
+
+ private:
+  std::vector<TensorBasePtr> tensor_list_;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_INC_TENSOR_IMP_H

mindspore_serving/ccsrc/common/tensor_base.cc

@@ -0,0 +1,147 @@
+#include "tensor_base.h"
+#include <functional>
+#include "log.h"
+#include "securec.h"
+
+#define TENSOR_MAX_ELEMENT_COUNT UINT32_MAX
+
+namespace mindspore::serving {
+
+bool TensorBase::set_data(const void *data, size_t data_len) {
+  resize_data(data_len);
+  if (mutable_data() == nullptr) {
+    MSI_LOG_ERROR << "set data failed, data len " << data_len;
+    return false;
+  }
+  if (data_size() != data_len) {
+    MSI_LOG_ERROR << "set data failed, tensor current data size " << data_size() << " not match data len " << data_len;
+    return false;
+  }
+  if (data_len == 0) {
+    return true;
+  }
+  memcpy_s(mutable_data(), data_size(), data, data_len);
+  return true;
+}
+
+size_t TensorBase::itemsize() const { return GetTypeSize(data_type()); }
+
+size_t TensorBase::element_cnt() const {
+  size_t element_num = 1;
+  for (auto dim : shape()) {
+    if (dim <= 0 || TENSOR_MAX_ELEMENT_COUNT / static_cast<size_t>(dim) < element_num) {
+      return 0;
+    }
+    element_num *= dim;
+  }
+  return element_num;
+}
+
+size_t TensorBase::GetTypeSize(DataType type) {
+  const std::map<DataType, size_t> type_size_map{
+    {kMSI_Bool, sizeof(bool)},       {kMSI_Float64, sizeof(double)},   {kMSI_Int8, sizeof(int8_t)},
+    {kMSI_Uint8, sizeof(uint8_t)},   {kMSI_Int16, sizeof(int16_t)},    {kMSI_Uint16, sizeof(uint16_t)},
+    {kMSI_Int32, sizeof(int32_t)},   {kMSI_Uint32, sizeof(uint32_t)},  {kMSI_Int64, sizeof(int64_t)},
+    {kMSI_Uint64, sizeof(uint64_t)}, {kMSI_Float16, sizeof(uint16_t)}, {kMSI_Float32, sizeof(float)},
+  };
+  auto it = type_size_map.find(type);
+  if (it != type_size_map.end()) {
+    return it->second;
+  }
+  return 0;
+}
+
+void TensorBase::assgin(const TensorBase &other) {
+  if (is_bytes_val_data()) {
+    clear_bytes_data();
+  }
+  set_shape(other.shape());
+  set_data_type(other.data_type());
+  if (other.is_bytes_val_data()) {
+    for (size_t i = 0; i < other.bytes_data_size(); i++) {
+      const uint8_t *data;
+      size_t data_len;
+      other.get_bytes_data(i, data, data_len);
+      add_bytes_data(data, data_len);
+    }
+  } else {
+    set_data(other.data(), other.data_size());
+  }
+}
+Status TensorBase::concat(const std::vector<TensorBasePtr> &inputs) {
+  if (inputs.empty()) {
+    MSI_LOG_ERROR << "inputs is empty";
+    return FAILED;
+  }
+  if (is_bytes_val_data()) {
+    clear_bytes_data();
+  }
+  auto &input0 = inputs[0];
+  std::vector<int64_t> new_shape = input0->shape();
+  new_shape.insert(new_shape.begin(), static_cast<int64_t>(inputs.size()));
+  set_shape(new_shape);
+  set_data_type(input0->data_type());
+  if (input0->is_bytes_val_data()) {
+    if (input0->bytes_data_size() != 1) {
+      MSI_LOG_ERROR << "input 0 bytes data batch size " << input0->bytes_data_size() << " is not 1";
+      return FAILED;
+    }
+    const uint8_t *data;
+    size_t data_len;
+    input0->get_bytes_data(0, data, data_len);
+    add_bytes_data(data, data_len);
+  } else {
+    resize_data(input0->data_size() * inputs.size());
+    memcpy_s(mutable_data(), data_size(), input0->data(), input0->data_size());
+  }
+  for (size_t i = 1; i < inputs.size(); i++) {
+    auto &other = inputs[i];
+    if (input0->data_type() != other->data_type()) {
+      MSI_LOG_ERROR << "input " << i << " data type " << other->data_type() << " not match input 0 "
+                    << input0->data_type();
+      return FAILED;
+    }
+    if (input0->shape() != other->shape()) {
+      MSI_LOG_ERROR << "input " << i << " shape " << other->shape() << " not match input 0 " << input0->shape();
+      return FAILED;
+    }
+    if (input0->is_bytes_val_data()) {
+      if (other->bytes_data_size() != 1) {
+        MSI_LOG_ERROR << "input " << i << " bytes data batch size " << other->bytes_data_size() << " is not 1";
+        return FAILED;
+      }
+      const uint8_t *data = nullptr;
+      size_t data_len = 0;
+      other->get_bytes_data(0, data, data_len);
+      add_bytes_data(data, data_len);
+    } else {
+      if (input0->data_size() != other->data_size()) {
+        MSI_LOG_ERROR << "input " << i << " data size " << other->data_size() << " not match input 0 "
+                      << input0->data_size();
+        return FAILED;
+      }
+      memcpy_s(mutable_data() + input0->data_size() * i, data_size() - input0->data_size() * i, other->data(),
+               other->data_size());
+    }
+  }
+  return SUCCESS;
+}
+
+LogStream &operator<<(LogStream &stream, DataType data_type) {
+  const std::map<DataType, std::string> type_name_map{
+    {kMSI_Unknown, "kMSI_Unknown"}, {kMSI_Bool, "kMSI_Bool"},       {kMSI_Int8, "kMSI_Int8"},
+    {kMSI_Uint8, "kMSI_Uint8"},     {kMSI_Int16, "kMSI_Int16"},     {kMSI_Uint16, "kMSI_Uint16"},
+    {kMSI_Int32, "kMSI_Int32"},     {kMSI_Uint32, "kMSI_Uint32"},   {kMSI_Int64, "kMSI_Int64"},
+    {kMSI_Uint64, "kMSI_Uint64"},   {kMSI_Float16, "kMSI_Float16"}, {kMSI_Float32, "kMSI_Float32"},
+    {kMSI_Float64, "kMSI_Float64"}, {kMSI_Bytes, "kMSI_Bytes"},     {kMSI_String, "kMSI_String"},
+  };
+  auto it = type_name_map.find(data_type);
+  if (it != type_name_map.end()) {
+    stream << it->second;
+  } else {
+    stream << "kMSI_Unknown";
+  }
+  return stream;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/tensor_base.h

@@ -0,0 +1,116 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_INC_TENSOR_H
+#define MINDSPORE_SERVING_INC_TENSOR_H
+
+#include <utility>
+#include <vector>
+#include <memory>
+#include <numeric>
+#include <map>
+#include <functional>
+#include <cstring>
+#include "log.h"
+#include "status.h"
+
+namespace mindspore {
+namespace serving {
+
+enum DataType {
+  kMSI_Unknown = 0,
+  kMSI_Bool = 1,
+  kMSI_Int8 = 2,
+  kMSI_Int16 = 3,
+  kMSI_Int32 = 4,
+  kMSI_Int64 = 5,
+  kMSI_Uint8 = 6,
+  kMSI_Uint16 = 7,
+  kMSI_Uint32 = 8,
+  kMSI_Uint64 = 9,
+  kMSI_Float16 = 10,
+  kMSI_Float32 = 11,
+  kMSI_Float64 = 12,
+  kMSI_String = 13,  // for model STRING input
+  kMSI_Bytes = 14,   // for image etc.
+};
+
+class TensorBase;
+using TensorBasePtr = std::shared_ptr<TensorBase>;
+
+class MS_API TensorBase {
+ public:
+  TensorBase() = default;
+  virtual ~TensorBase() = default;
+
+  // For all data type
+  virtual std::vector<int64_t> shape() const = 0;
+  virtual void set_shape(const std::vector<int64_t> &shape) = 0;
+  virtual DataType data_type() const = 0;
+  virtual void set_data_type(DataType type) = 0;
+
+  // All the following interfaces are not for  kMSI_String and kMSI_Bytes
+  virtual const uint8_t *data() const = 0;
+  virtual size_t data_size() const = 0;
+  virtual bool resize_data(size_t data_len) = 0;
+  virtual uint8_t *mutable_data() = 0;
+
+  // Byte size of a single element.
+  size_t itemsize() const;
+  // Total number of elements.
+  size_t element_cnt() const;
+  // resize and copy data
+  bool set_data(const void *data, size_t data_len);
+  static size_t GetTypeSize(DataType type);
+
+  // For kMSI_String and kMSI_Bytes
+  virtual void clear_bytes_data() = 0;
+  virtual void add_bytes_data(const uint8_t *data, size_t bytes_len) = 0;
+  virtual size_t bytes_data_size() const = 0;
+  virtual void get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const = 0;
+
+  // TensorBase(const TensorBase& other) = delete;
+  // TensorBase& operator=(const TensorBase& other) = delete;
+  void assgin(const TensorBase &other);
+  Status concat(const std::vector<TensorBasePtr> &inputs);
+  bool is_bytes_val_data() const { return data_type() == kMSI_Bytes || data_type() == kMSI_String; }
+};
+
+class RequestBase {
+ public:
+  RequestBase() = default;
+  virtual ~RequestBase() = default;
+  virtual size_t size() const = 0;
+  virtual const TensorBase *operator[](size_t index) const = 0;
+};
+
+class ReplyBase {
+ public:
+  ReplyBase() = default;
+  virtual ~ReplyBase() = default;
+  virtual size_t size() const = 0;
+  virtual TensorBase *operator[](size_t index) = 0;
+  virtual const TensorBase *operator[](size_t index) const = 0;
+  virtual TensorBase *add() = 0;
+  virtual void clear() = 0;
+};
+
+extern MS_API LogStream &operator<<(LogStream &stream, DataType data_type);
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVING_INC_TENSOR_H

mindspore_serving/ccsrc/common/thread_pool.cc

@@ -0,0 +1,69 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/thread_pool.h"
+#include <atomic>
+#include <functional>
+#include <queue>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+namespace mindspore::serving {
+
+ThreadPool::ThreadPool(uint32_t size) : is_stoped_(false), idle_thrd_num_(size < 1 ? 1 : size) {
+  for (uint32_t i = 0; i < idle_thrd_num_; ++i) {
+    pool_.emplace_back(ThreadFunc, this);
+  }
+}
+
+ThreadPool::~ThreadPool() {
+  is_stoped_.store(true);
+  cond_var_.notify_all();
+
+  for (std::thread &thd : pool_) {
+    if (thd.joinable()) {
+      try {
+        thd.join();
+      } catch (const std::system_error &) {
+      } catch (...) {
+      }
+    }
+  }
+}
+
+void ThreadPool::ThreadFunc(ThreadPool *thread_pool) {
+  if (thread_pool == nullptr) {
+    return;
+  }
+  while (!thread_pool->is_stoped_) {
+    std::function<void()> task;
+    {
+      std::unique_lock<std::mutex> lock{thread_pool->m_lock_};
+      thread_pool->cond_var_.wait(
+        lock, [thread_pool] { return thread_pool->is_stoped_.load() || !thread_pool->tasks_.empty(); });
+      if (thread_pool->is_stoped_ && thread_pool->tasks_.empty()) {
+        return;
+      }
+      task = std::move(thread_pool->tasks_.front());
+      thread_pool->tasks_.pop();
+    }
+    --thread_pool->idle_thrd_num_;
+    task();
+    ++thread_pool->idle_thrd_num_;
+  }
+}
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/common/thread_pool.h

@@ -0,0 +1,74 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MS_SERVING_UTILS_THREAD_POOL_H_
+#define MS_SERVING_UTILS_THREAD_POOL_H_
+
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <future>
+#include <memory>
+#include <queue>
+#include <stdexcept>
+#include <thread>
+#include <utility>
+#include <vector>
+
+namespace mindspore::serving {
+using ThreadTask = std::function<void()>;
+
+class ThreadPool {
+ public:
+  explicit ThreadPool(uint32_t size = 4);
+
+  ~ThreadPool();
+
+  template <class Func, class... Args>
+  auto commit(Func &&func, Args &&... args) -> std::future<decltype(func(args...))> {
+    using retType = decltype(func(args...));
+    std::future<retType> fail_future;
+    if (is_stoped_.load()) {
+      return fail_future;
+    }
+
+    auto bindFunc = std::bind(std::forward<Func>(func), std::forward<Args>(args)...);
+    auto task = std::make_shared<std::packaged_task<retType()>>(bindFunc);
+    if (task == nullptr) {
+      return fail_future;
+    }
+    std::future<retType> future = task->get_future();
+    {
+      std::lock_guard<std::mutex> lock{m_lock_};
+      tasks_.emplace([task]() { (*task)(); });
+    }
+    cond_var_.notify_one();
+    return future;
+  }
+
+  static void ThreadFunc(ThreadPool *thread_pool);
+
+ private:
+  std::vector<std::thread> pool_;
+  std::queue<ThreadTask> tasks_;
+  std::mutex m_lock_;
+  std::condition_variable cond_var_;
+  std::atomic<bool> is_stoped_;
+  std::atomic<uint32_t> idle_thrd_num_;
+};
+}  // namespace mindspore::serving
+
+#endif  // MS_SERVING_UTILS_THREAD_POOL_H_

mindspore_serving/ccsrc/master/dispacther.cc

@@ -0,0 +1,346 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "master/dispacther.h"
+#include "worker/worker.h"
+#include "common/proto_tensor.h"
+
+namespace mindspore::serving {
+
+Dispatcher::Dispatcher() {}
+
+Dispatcher::~Dispatcher() { Clear(); }
+
+DispatcherWorkerContext Dispatcher::GetWorkSession(const RequestSpec &request_spec) const {
+  Status status;
+  DispatcherWorkerContext context;
+  auto it = servable_map_.find(request_spec.servable_name);
+  if (it == servable_map_.end()) {
+    return context;
+  }
+  if (request_spec.version_number > 0) {
+    auto item = find_if(it->second.begin(), it->second.end(), [&](const DispatcherWorkerContext &v) {
+      return v.worker_spec.version_number == request_spec.version_number;
+    });
+    if (item != it->second.end()) {
+      context.worker_spec = item->worker_spec;
+      context.stub_ = item->stub_;
+      context.worker_running_in_master = item->worker_running_in_master;
+    }
+    return context;
+  }
+  uint64_t max_version_number = 0;
+  for (const auto &item : it->second) {
+    if (max_version_number < item.worker_spec.version_number) {
+      context.worker_spec = item.worker_spec;
+      context.stub_ = item.stub_;
+      context.worker_running_in_master = item.worker_running_in_master;
+      max_version_number = item.worker_spec.version_number;
+    }
+  }
+  return context;
+}
+
+Status Dispatcher::Dispatch(const proto::PredictRequest &request, proto::PredictReply &reply) {
+  std::shared_lock<std::shared_mutex> lock(servable_shared_lock_);
+  RequestSpec request_spec;
+  GrpcTensorHelper::GetRequestSpec(request, request_spec);
+  auto worker = GetWorkSession(request_spec);
+  if (!worker.stub_ && !worker.worker_running_in_master) {
+    return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Request " << request_spec.Repr() << ", servable is not available";
+  }
+  bool find_method =
+    std::any_of(worker.worker_spec.methods.begin(), worker.worker_spec.methods.end(),
+                [&](const WorkerMethodInfo &method) { return method.name == request_spec.method_name; });
+  if (!find_method) {
+    return INFER_STATUS_LOG_ERROR(INVALID_INPUTS) << "Request " << request_spec.Repr() << ", method is not available";
+  }
+  /// TODO spec request version_number
+  if (worker.stub_ != nullptr) {
+    grpc::ClientContext context;
+    auto status = worker.stub_->Predict(&context, request, &reply);
+    if (!status.ok()) {
+      return INFER_STATUS_LOG_ERROR(FAILED)
+             << "Predict failed, worker gRPC error: " << status.error_code() << ", " << status.error_message();
+    }
+  } else {
+    return Worker::GetInstance().Run(request, reply);
+  }
+  return SUCCESS;
+}
+
+Status Dispatcher::RegisterServable(const proto::RegisterRequest &request, proto::RegisterReply &reply) {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  std::vector<WorkerSpec> worker_specs;
+  GrpcTensorHelper::GetWorkerSpec(request, worker_specs);
+  if (worker_specs.empty()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Register failed, servable cannot be empty";
+  }
+  for (auto &worker_spec : worker_specs) {
+    if (worker_spec.servable_name.empty()) {
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Register failed, servable name cannot be empty";
+    }
+    if (worker_spec.version_number <= 0) {
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Register failed, servable name " << worker_spec.servable_name
+                                            << " version number " << worker_spec.version_number << " cannot be 0";
+    }
+    auto target_str = request.address();
+    auto it = servable_map_.find(worker_spec.servable_name);
+
+    std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials());
+
+    bool find_registered = false;
+    if (it != servable_map_.end()) {
+      std::shared_ptr<Worker> worker;
+      auto item = find_if(it->second.begin(), it->second.end(), [&](const DispatcherWorkerContext &v) {
+        return v.worker_spec.version_number == worker_spec.version_number &&
+               v.worker_spec.worker_address == worker_spec.worker_address;
+      });
+      if (item != it->second.end()) {
+        MSI_LOG_WARNING << "Servable " << worker_spec.servable_name << " version " << worker_spec.version_number
+                        << " has been registered, old registered info will be replaced";
+        item->worker_spec = worker_spec;
+        item->stub_ = proto::MSWorker::NewStub(channel);
+        find_registered = true;
+      }
+    }
+    if (!find_registered) {
+      DispatcherWorkerContext context;
+      context.worker_spec = worker_spec;
+      context.stub_ = proto::MSWorker::NewStub(channel);
+      servable_map_[worker_spec.servable_name].push_back(context);
+    }
+  }
+  return SUCCESS;
+}
+
+Status Dispatcher::UnregisterServable(const proto::ExitRequest &request, proto::ExitReply &reply) {
+  if (clearing_flag) {
+    return SUCCESS;
+  }
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  auto target_str = request.address();
+  Status status;
+  for (auto iter = servable_map_.begin(); iter != servable_map_.end();) {
+    for (auto it = iter->second.begin(); it != iter->second.end();) {
+      if (target_str == it->worker_spec.worker_address) {
+        it = iter->second.erase(it);
+      } else {
+        ++it;
+      }
+    }
+    if (iter->second.size() == 0) {
+      iter = servable_map_.erase(iter);
+    } else {
+      ++iter;
+    }
+  }
+  return SUCCESS;
+}
+Status Dispatcher::AddServable(const proto::AddWorkerRequest &request, proto::AddWorkerReply &reply) {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  WorkerSpec worker_spec;
+  GrpcTensorHelper::GetWorkerSpec(request, worker_spec);
+  auto target_str = request.address();
+  if (worker_spec.servable_name.empty()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "AddServable failed, servable name cannot be empty";
+  }
+  if (worker_spec.version_number <= 0) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "AddServable failed, servable name " << worker_spec.servable_name
+                                          << " version number " << worker_spec.version_number << " cannot be 0";
+  }
+  Status status;
+  auto it = servable_map_.find(worker_spec.servable_name);
+  if (it != servable_map_.end()) {
+    bool find = std::any_of(it->second.begin(), it->second.end(), [&](const DispatcherWorkerContext &item) {
+      return item.worker_spec.version_number == worker_spec.version_number &&
+             item.worker_spec.worker_address == worker_spec.worker_address;
+    });
+    if (find) {
+      MSI_LOG_WARNING << "Servable " << worker_spec.servable_name << " version " << worker_spec.version_number
+                      << " has been registered";
+      return SUCCESS;
+    }
+  }
+  DispatcherWorkerContext context;
+  context.worker_spec = worker_spec;
+  std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials());
+  context.stub_ = proto::MSWorker::NewStub(channel);
+  servable_map_[worker_spec.servable_name].push_back(context);
+  return SUCCESS;
+}
+
+Status Dispatcher::RemoveServable(const proto::RemoveWorkerRequest &request, proto::RemoveWorkerReply &reply) {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  WorkerSpec worker_spec;
+  GrpcTensorHelper::GetWorkerSpec(request, worker_spec);
+  auto target_str = request.address();
+  Status status;
+  for (auto iter = servable_map_.begin(); iter != servable_map_.end();) {
+    for (auto it = iter->second.begin(); it != iter->second.end();) {
+      if (target_str == it->worker_spec.worker_address && it->worker_spec.servable_name == worker_spec.servable_name &&
+          it->worker_spec.version_number == worker_spec.version_number) {
+        it = iter->second.erase(it);
+      } else {
+        ++it;
+      }
+    }
+    if (iter->second.size() == 0) {
+      iter = servable_map_.erase(iter);
+    } else {
+      ++iter;
+    }
+  }
+  return SUCCESS;
+}
+
+void Dispatcher::Clear() {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  clearing_flag = true;
+
+  for (auto iter = servable_map_.begin(); iter != servable_map_.end(); ++iter) {
+    for (auto it = iter->second.begin(); it != iter->second.end(); ++it) {
+      proto::ExitRequest request;
+      request.set_address(it->worker_spec.worker_address);
+      proto::ExitReply reply;
+      grpc::ClientContext context;
+      const int32_t TIME_OUT = 1;
+      std::chrono::system_clock::time_point deadline =
+        std::chrono::system_clock::now() + std::chrono::seconds(TIME_OUT);
+      context.set_deadline(deadline);
+      if (it->stub_) {
+        (void)it->stub_->Exit(&context, request, &reply);
+      } else {
+        Worker::GetInstance().Clear();
+      }
+    }
+  }
+  servable_map_.clear();
+}
+
+Status Dispatcher::RegisterLocalServable(const std::vector<WorkerSpec> &worker_specs) {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  if (worker_specs.empty()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Register failed, servable cannot be empty";
+  }
+  for (auto &worker_spec : worker_specs) {
+    if (worker_spec.servable_name.empty()) {
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Register failed, servable name cannot be empty";
+    }
+    if (worker_spec.version_number <= 0) {
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Register failed, servable name " << worker_spec.servable_name
+                                            << " version number " << worker_spec.version_number << " cannot be 0";
+    }
+    auto it = servable_map_.find(worker_spec.servable_name);
+
+    bool find_registered = false;
+    if (it != servable_map_.end()) {
+      std::shared_ptr<Worker> worker;
+      auto item = find_if(it->second.begin(), it->second.end(), [&](const DispatcherWorkerContext &v) {
+        return v.worker_spec.version_number == worker_spec.version_number &&
+               v.worker_spec.worker_address == worker_spec.worker_address;
+      });
+      if (item != it->second.end()) {
+        MSI_LOG_WARNING << "Servable " << worker_spec.servable_name << " version " << worker_spec.version_number
+                        << " has been registered, old registered info will be replaced";
+        item->worker_spec = worker_spec;
+        find_registered = true;
+      }
+    }
+    if (!find_registered) {
+      DispatcherWorkerContext context;
+      context.worker_spec = worker_spec;
+      context.worker_running_in_master = true;
+      servable_map_[worker_spec.servable_name].push_back(context);
+    }
+  }
+  return SUCCESS;
+}
+
+Status Dispatcher::UnregisterLocalServable() {
+  if (clearing_flag) {
+    return SUCCESS;
+  }
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  Status status;
+  for (auto iter = servable_map_.begin(); iter != servable_map_.end();) {
+    for (auto it = iter->second.begin(); it != iter->second.end();) {
+      if (it->worker_running_in_master) {
+        it = iter->second.erase(it);
+      } else {
+        ++it;
+      }
+    }
+    if (iter->second.size() == 0) {
+      iter = servable_map_.erase(iter);
+    } else {
+      ++iter;
+    }
+  }
+  return SUCCESS;
+}
+
+Status Dispatcher::AddLocalServable(const WorkerSpec &worker_spec) {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  if (worker_spec.servable_name.empty()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "AddServable failed, servable name cannot be empty";
+  }
+  if (worker_spec.version_number <= 0) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "AddServable failed, servable name " << worker_spec.servable_name
+                                          << " version number " << worker_spec.version_number << " cannot be 0";
+  }
+  Status status;
+  auto it = servable_map_.find(worker_spec.servable_name);
+  if (it != servable_map_.end()) {
+    bool find = std::any_of(it->second.begin(), it->second.end(), [&](const DispatcherWorkerContext &item) {
+      return item.worker_spec.version_number == worker_spec.version_number &&
+             item.worker_spec.worker_address == worker_spec.worker_address;
+    });
+    if (find) {
+      MSI_LOG_WARNING << "Servable " << worker_spec.servable_name << " version " << worker_spec.version_number
+                      << " has been registered";
+      return SUCCESS;
+    }
+  }
+  DispatcherWorkerContext context;
+  context.worker_spec = worker_spec;
+  context.worker_running_in_master = true;
+  servable_map_[worker_spec.servable_name].push_back(context);
+  return SUCCESS;
+}
+
+Status Dispatcher::RemoveLocalServable(const WorkerSpec &worker_spec) {
+  std::unique_lock<std::shared_mutex> lock(servable_shared_lock_);
+  Status status;
+  for (auto iter = servable_map_.begin(); iter != servable_map_.end();) {
+    for (auto it = iter->second.begin(); it != iter->second.end();) {
+      if (it->worker_running_in_master && it->worker_spec.servable_name == worker_spec.servable_name &&
+          it->worker_spec.version_number == worker_spec.version_number) {
+        it = iter->second.erase(it);
+      } else {
+        ++it;
+      }
+    }
+    if (iter->second.size() == 0) {
+      iter = servable_map_.erase(iter);
+    } else {
+      ++iter;
+    }
+  }
+  return SUCCESS;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/master/dispacther.h

@@ -0,0 +1,71 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_SERVER_DISPACTHER_H
+#define MINDSPORE_SERVING_SERVER_DISPACTHER_H
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <shared_mutex>
+#include "proto/ms_worker.grpc.pb.h"
+#include "common/serving_common.h"
+#include "common/instance.h"
+#include "common/servable.h"
+#include "worker/worker.h"
+
+namespace mindspore::serving {
+
+using DispatchCallback = std::function<Status(const std::vector<Instance> &outputs)>;
+
+struct DispatcherWorkerContext {
+  WorkerSpec worker_spec;
+  std::shared_ptr<proto::MSWorker::Stub> stub_ = nullptr;
+  bool worker_running_in_master = false;
+};
+
+class MS_API Dispatcher {
+ public:
+  Dispatcher();
+  ~Dispatcher();
+  Status Dispatch(const proto::PredictRequest &request, proto::PredictReply &reply);
+
+  Status RegisterServable(const proto::RegisterRequest &request, proto::RegisterReply &reply);
+  Status UnregisterServable(const proto::ExitRequest &request, proto::ExitReply &reply);
+
+  Status AddServable(const proto::AddWorkerRequest &request, proto::AddWorkerReply &reply);
+  Status RemoveServable(const proto::RemoveWorkerRequest &request, proto::RemoveWorkerReply &reply);
+
+  void Clear();
+
+  Status RegisterLocalServable(const std::vector<WorkerSpec> &worker_specs);
+  Status UnregisterLocalServable();
+  Status AddLocalServable(const WorkerSpec &worker_spec);
+  Status RemoveLocalServable(const WorkerSpec &worker_spec);
+
+ private:
+  std::unordered_map<std::string, std::vector<DispatcherWorkerContext>> servable_map_{};
+  std::shared_mutex servable_shared_lock_;
+  // avoid invoke Clear and then UnregisterServable is invoked by Clear in other thread
+  std::atomic_bool clearing_flag = false;
+
+  DispatcherWorkerContext GetWorkSession(const RequestSpec &request_spec) const;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_SERVER_DISPACTHER_H

mindspore_serving/ccsrc/master/grpc/grpc_process.cc

@@ -0,0 +1,153 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "master/grpc/grpc_process.h"
+#include "master/dispacther.h"
+
+namespace mindspore {
+namespace serving {
+
+namespace {
+std::string GetProtorWorkerSpecRepr(const proto::WorkerSpec &worker_spec) {
+  std::stringstream str;
+  str << "{name:" << worker_spec.name() << ", version:" << worker_spec.version_number() << ", method:[";
+  for (int k = 0; k < worker_spec.methods_size(); k++) {
+    str << worker_spec.methods(k).name();
+    if (k + 1 < worker_spec.methods_size()) {
+      str << ",";
+    }
+  }
+  str << "]}";
+  return str.str();
+}
+}  // namespace
+
+grpc::Status MSServiceImpl::Predict(grpc::ServerContext *context, const proto::PredictRequest *request,
+                                    proto::PredictReply *reply) {
+  Status status(FAILED);
+  try {
+    MSI_TIME_STAMP_START(Predict)
+    status = dispatcher_->Dispatch(*request, *reply);
+    MSI_TIME_STAMP_END(Predict)
+  } catch (const std::bad_alloc &ex) {
+    MSI_LOG(ERROR) << "Serving Error: malloc memory failed";
+    std::cout << "Serving Error: malloc memory failed" << std::endl;
+  } catch (const std::runtime_error &ex) {
+    MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what();
+    std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl;
+  } catch (const std::exception &ex) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what();
+    std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl;
+  } catch (...) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred";
+    std::cout << "Serving Error: exception occurred";
+  }
+  MSI_LOG(INFO) << "Finish call service Eval";
+
+  if (status != SUCCESS) {
+    auto proto_error_msg = reply->add_error_msg();
+    proto_error_msg->set_error_code(FAILED);
+    std::string error_msg = status.StatusMessage();
+    if (error_msg.empty()) {
+      proto_error_msg->set_error_msg("Predict failed");
+    } else {
+      proto_error_msg->set_error_msg(error_msg);
+    }
+    return grpc::Status::OK;
+  }
+  return grpc::Status::OK;
+}
+
+grpc::Status MSMasterImpl::Register(grpc::ServerContext *context, const proto::RegisterRequest *request,
+                                    proto::RegisterReply *reply) {
+  auto worker_sig = [request]() {
+    std::stringstream str;
+    str << "worker address: " << request->address() << ", servables: [";
+    for (int i = 0; i < request->worker_spec_size(); i++) {
+      str << GetProtorWorkerSpecRepr(request->worker_spec(i));
+      if (i + 1 < request->worker_spec_size()) {
+        str << ", ";
+      }
+    }
+    str << "]";
+    return str.str();
+  };
+  Status status(FAILED);
+  status = dispatcher_->RegisterServable(*request, *reply);
+  if (status != SUCCESS) {
+    MSI_LOG_ERROR << "Register servable failed, " << worker_sig();
+    return grpc::Status::OK;
+  }
+  MSI_LOG(INFO) << "Register success: " << worker_sig();
+  return grpc::Status::OK;
+}
+
+grpc::Status MSMasterImpl::AddWorker(grpc::ServerContext *context, const proto::AddWorkerRequest *request,
+                                     proto::AddWorkerReply *reply) {
+  auto worker_sig = [request]() {
+    std::stringstream str;
+    str << "worker address: " << request->address()
+        << ", servables: " << GetProtorWorkerSpecRepr(request->worker_spec());
+    return str.str();
+  };
+  Status status(FAILED);
+  status = dispatcher_->AddServable(*request, *reply);
+  if (status != SUCCESS) {
+    MSI_LOG_ERROR << "Add servable failed, " << worker_sig();
+    return grpc::Status::OK;
+  }
+  MSI_LOG(INFO) << "Add success, " << worker_sig();
+  return grpc::Status::OK;
+}
+
+grpc::Status MSMasterImpl::RemoveWorker(grpc::ServerContext *context, const proto::RemoveWorkerRequest *request,
+                                        proto::RemoveWorkerReply *reply) {
+  auto worker_sig = [request]() {
+    std::stringstream str;
+    str << "worker address: " << request->address()
+        << ", servables: " << GetProtorWorkerSpecRepr(request->worker_spec());
+    return str.str();
+  };
+  Status status(FAILED);
+  status = dispatcher_->RemoveServable(*request, *reply);
+  if (status != SUCCESS) {
+    MSI_LOG_ERROR << "Add servable failed, " << worker_sig();
+    return grpc::Status::OK;
+  }
+  MSI_LOG(INFO) << "Add success, " << worker_sig();
+  return grpc::Status::OK;
+}
+
+grpc::Status MSMasterImpl::Exit(grpc::ServerContext *context, const proto::ExitRequest *request,
+                                proto::ExitReply *reply) {
+  auto worker_sig = [request]() {
+    std::stringstream str;
+    str << "worker address: " << request->address();
+    return str.str();
+  };
+
+  MSI_LOG(INFO) << "Worker Exit, " << worker_sig();
+  Status status(FAILED);
+  status = dispatcher_->UnregisterServable(*request, *reply);
+  if (status != SUCCESS) {
+    MSI_LOG_ERROR << "UnRegister servable failed, " << worker_sig();
+    return grpc::Status::OK;
+  }
+  return grpc::Status::OK;
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/master/grpc/grpc_process.h

@@ -0,0 +1,68 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_GRPC_PROCESS_H
+#define MINDSPORE_GRPC_PROCESS_H
+
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/health_check_service_interface.h>
+#include <grpcpp/ext/proto_server_reflection_plugin.h>
+#include <memory>
+#include "common/serving_common.h"
+#include "proto/ms_service.pb.h"
+#include "proto/ms_service.grpc.pb.h"
+#include "proto/ms_master.pb.h"
+#include "proto/ms_master.grpc.pb.h"
+#include "master/dispacther.h"
+
+namespace mindspore {
+namespace serving {
+
+// Service Implement
+class MSServiceImpl final : public proto::MSService::Service {
+ public:
+  explicit MSServiceImpl(std::shared_ptr<Dispatcher> dispatcher) : dispatcher_(dispatcher) {}
+  ~MSServiceImpl() = default;
+
+  grpc::Status Predict(grpc::ServerContext *context, const proto::PredictRequest *request,
+                       proto::PredictReply *reply) override;
+
+ private:
+  std::shared_ptr<Dispatcher> dispatcher_;
+};
+
+// Service Implement
+class MSMasterImpl final : public proto::MSMaster::Service {
+ public:
+  explicit MSMasterImpl(std::shared_ptr<Dispatcher> dispatcher) : dispatcher_(dispatcher) {}
+  ~MSMasterImpl() = default;
+
+  grpc::Status Register(grpc::ServerContext *context, const proto::RegisterRequest *request,
+                        proto::RegisterReply *reply) override;
+  grpc::Status Exit(grpc::ServerContext *context, const proto::ExitRequest *request, proto::ExitReply *reply) override;
+  grpc::Status AddWorker(grpc::ServerContext *context, const proto::AddWorkerRequest *request,
+                         proto::AddWorkerReply *reply) override;
+  grpc::Status RemoveWorker(grpc::ServerContext *context, const proto::RemoveWorkerRequest *request,
+                            proto::RemoveWorkerReply *reply) override;
+
+ private:
+  std::shared_ptr<Dispatcher> dispatcher_;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_GRPC_PROCESS_H

mindspore_serving/ccsrc/master/restful/http_handle.cc

@@ -0,0 +1,146 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "master/restful/http_handle.h"
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <vector>
+
+#include "master/restful/http_process.h"
+#include "master/server.h"
+
+namespace mindspore {
+namespace serving {
+
+static std::vector<unsigned char> encode_table = {
+  'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+  'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
+  's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
+static std::vector<unsigned char> decode_table = {
+  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62,
+  255, 255, 255, 63,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  255, 255, 255, 255, 255, 255, 255, 0,
+  1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,  13,  14,  15,  16,  17,  18,  19,  20,  21,  22,
+  23,  24,  25,  255, 255, 255, 255, 255, 255, 26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,
+  39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,  255, 255, 255, 255, 255};
+
+size_t Base64Encode(const uint8_t *input, size_t length, uint8_t *output) {
+  if (length == 0) {
+    return 0;
+  }
+
+  size_t i, j;
+  for (i = 0, j = 0; i + 3 <= length; i += 3) {
+    output[j++] = encode_table[input[i] >> 2];
+    output[j++] = encode_table[((input[i] << 4) & 0x30) | (input[i + 1] >> 4)];
+    output[j++] = encode_table[((input[i + 1] << 2) & 0x3c) | (input[i + 2] >> 6)];
+    output[j++] = encode_table[input[i + 2] & 0x3f];
+  }
+
+  if (i < length) {
+    uint32_t left_num = length - i;
+    if (left_num == 1) {
+      output[j++] = encode_table[input[i] >> 2];
+      output[j++] = encode_table[(input[i] << 4) & 0x30];
+      output[j++] = '=';
+      output[j++] = '=';
+    } else {
+      output[j++] = encode_table[input[i] >> 2];
+      output[j++] = encode_table[((input[i] << 4) & 0x30) | (input[i + 1] >> 4)];
+      output[j++] = encode_table[(input[i + 1] << 2) & 0x3c];
+      output[j++] = '=';
+    }
+  }
+  return j;
+}
+
+size_t Base64Decode(const uint8_t *target, size_t target_length, uint8_t *origin) {
+  if (target_length == 0 || target_length % 4 != 0) {
+    return 0;
+  }
+  size_t i, j = 0;
+  uint8_t value[4];
+  for (i = 0; i < target_length; i += 4) {
+    for (size_t k = 0; k < 4; k++) {
+      value[k] = decode_table[target[i + k]];
+    }
+
+    // value[2], value[3]:may be '='
+    if (value[0] >= 64 || value[1] >= 64) {
+      MSI_LOG_EXCEPTION << "Decode value is not more than max value 64";
+    }
+
+    origin[j++] = (value[0] << 2) | (value[1] >> 4);
+
+    if (value[2] >= 64) {
+      break;
+    } else if (value[3] >= 64) {
+      origin[j++] = (value[1] << 4) | (value[2] >> 2);
+      break;
+    } else {
+      origin[j++] = (value[1] << 4) | (value[2] >> 2);
+      origin[j++] = (value[2] << 6) | value[3];
+    }
+  }
+  return j;
+}
+
+size_t GetB64TargetSize(size_t origin_len) {
+  size_t target_size = 0;
+  if (origin_len % 3 == 0) {
+    target_size = (origin_len / 3) * 4;
+  } else {
+    target_size = (origin_len / 3 + 1) * 4;
+  }
+  return target_size;
+}
+
+size_t GetB64OriginSize(size_t target_len, size_t tail_size) {
+  size_t origin_length = 0;
+  if (target_len == 0 || target_len % 4 != 0) {
+    return origin_length;
+  }
+  origin_length = 3 * (target_len / 4) - tail_size;
+  return origin_length;
+}
+
+size_t GetTailEqualSize(const std::string &str) {
+  size_t length = str.size();
+  if (length == 0 || length % 4 != 0) {
+    return UINT32_MAX;
+  }
+  size_t count = 0;
+  if (str.at(str.size() - 1) == '=') {
+    count++;
+  }
+
+  if (str.at(str.size() - 2) == '=') {
+    count++;
+  }
+
+  return count;
+}
+
+Status HandleRestfulRequest(const std::shared_ptr<RestfulRequest> &restful_request, json *const out_json) {
+  Status status(SUCCESS);
+  std::shared_ptr<Dispatcher> dispatcher_ = Server::Instance().GetDispatcher();
+  RestfulService restful_service(dispatcher_);
+  status = restful_service.RunRestful(restful_request, out_json);
+  return status;
+}
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/master/restful/http_handle.h

@@ -0,0 +1,38 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_HTTP_HANDLE_H
+#define MINDSPORE_SERVING_HTTP_HANDLE_H
+
+#include <string>
+#include <memory>
+#include "common/serving_common.h"
+#include "master/restful/restful_request.h"
+
+using nlohmann::json;
+namespace mindspore {
+namespace serving {
+Status HandleRestfulRequest(const std::shared_ptr<RestfulRequest> &restful_request, json *const out_json);
+
+size_t Base64Encode(const uint8_t *input, size_t length, uint8_t *output);
+size_t Base64Decode(const uint8_t *target, size_t target_length, uint8_t *origin);
+size_t GetB64TargetSize(size_t origin_len);
+size_t GetB64OriginSize(size_t target_len, size_t tail_size);
+size_t GetTailEqualSize(const std::string &str);
+
+}  // namespace serving
+}  // namespace mindspore
+#endif  // MINDSPORE_SERVING_HTTP_HANDLE_H

mindspore_serving/ccsrc/master/restful/http_process.h

@@ -0,0 +1,110 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_HTTP_PROCESS_H
+#define MINDSPORE_SERVING_HTTP_PROCESS_H
+
+#include <string>
+#include <memory>
+#include <vector>
+#include <nlohmann/json.hpp>
+#include "proto/ms_service.pb.h"
+#include "master/dispacther.h"
+#include "common/proto_tensor.h"
+#include "master/restful/restful_request.h"
+
+using nlohmann::json;
+using std::string;
+
+namespace mindspore {
+namespace serving {
+constexpr auto kInstancesRequest = "instances";
+constexpr auto kInstancesReply = "instances";
+constexpr auto kErrorMsg = "error_msg";
+constexpr auto kType = "type";
+constexpr auto kShape = "shape";
+constexpr auto kB64 = "b64";
+
+enum RequestType { kInstanceType = 0, kInvalidType };
+enum InstancesType { kNokeyWay = 0, kKeyWay, kInvalidWay };
+enum HTTP_DATA_TYPE { HTTP_DATA_NONE, HTTP_DATA_INT, HTTP_DATA_FLOAT, HTTP_DATA_BOOL, HTTP_DATA_STR, HTTP_DATA_OBJ };
+class RestfulService {
+ public:
+  explicit RestfulService(std::shared_ptr<Dispatcher> dispatcher) : dispatcher_(dispatcher) {}
+  ~RestfulService() = default;
+  Status RunRestful(const std::shared_ptr<RestfulRequest> &restful_request, json *const out_json);
+
+ private:
+  Status CheckObjTypeMatchShape(DataType data_type, const std::vector<int64_t> &shape);
+  std::string GetString(const uint8_t *ptr, size_t length);
+  Status CheckObj(const json &js);
+  Status CheckObjType(const std::string &type);
+  DataType GetObjDataType(const json &js);
+  std::vector<int64_t> GetObjShape(const json &js);
+  std::vector<int64_t> GetArrayShape(const json &json_array);
+  std::vector<int64_t> GetSpecifiedShape(const json &js);
+  DataType GetArrayDataType(const json &json_array, HTTP_DATA_TYPE &type_format);
+  Status CheckReqJsonValid(const json &js_msg);
+  std::string GetStringByDataType(DataType type);
+  bool JsonMatchDataType(const json &js, DataType type);
+
+  template <typename T>
+  Status GetScalarData(const json &js, size_t index, bool is_bytes, ProtoTensor *const request_tensor);
+  Status GetScalarByType(DataType type, const json &js, size_t index, ProtoTensor *const request_tensor);
+
+  Status RecursiveGetArray(const json &json_data, size_t depth, size_t data_index, HTTP_DATA_TYPE type_format,
+                           ProtoTensor *const request_tensor);
+  Status GetArrayData(const json &js, size_t data_index, HTTP_DATA_TYPE type, ProtoTensor *const request_tensor);
+
+  Status ParseReqCommonMsg(const std::shared_ptr<RestfulRequest> &restful_request,
+                           proto::PredictRequest *const request);
+  Status ParseRequest(const std::shared_ptr<RestfulRequest> &restful_request, proto::PredictRequest *const request);
+  Status ParseInstancesMsg(const json &js_msg, proto::PredictRequest *const request);
+  Status GetInstancesType(const json &instances);
+  Status ParseKeyInstances(const json &instances, proto::PredictRequest *const request);
+  Status PaserKeyOneInstance(const json &instance_msg, proto::PredictRequest *const request);
+  Status ParseItem(const json &value, ProtoTensor *const pb_tensor);
+
+  // parse reply:trans RequestReply to http msg
+  RequestType GetReqType(const std::string &str);
+  std::string GetReqTypeStr(RequestType req_type);
+  Status ParseReply(const proto::PredictReply &reply, json *const out_json);
+  Status CheckReply(const ProtoTensor &pb_tensor);
+  Status ParseInstancesReply(const proto::PredictReply &reply, json *const out_json);
+  Status ParseReplyDetail(proto::Tensor tensor, json *const js);
+  Status ParseScalar(const ProtoTensor &pb_tensor, size_t index, json *const js);
+  Status RecursiveParseArray(ProtoTensor pb_tensor, size_t depth, size_t pos, json *const out_json);
+
+  template <typename T>
+  Status ParseScalarData(const ProtoTensor &pb_tensor, bool is_bytes, size_t index, json *const js);
+  template <typename T>
+  bool IsString();
+  void ParseErrorMsg(const proto::ErrorMsg &error_msg, json *const js);
+
+  void PrintRequest(const proto::PredictRequest *const request);
+  void PrintReply(const proto::PredictReply &reply);
+  void FadeReply(const proto::PredictRequest &request, proto::PredictReply &reply);
+
+  RequestType request_type_{kInvalidType};
+  InstancesType instances_type_{kInvalidWay};
+  int64_t instances_nums_{0};
+  std::shared_ptr<Dispatcher> dispatcher_;
+  std::vector<std::string> request_type_list_ = {kInstancesRequest};
+};
+
+}  // namespace serving
+}  // namespace mindspore
+#endif  // MINDSPORE_SERVER_H

mindspore_serving/ccsrc/master/restful/restful_request.cc

@@ -0,0 +1,208 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "master/restful/restful_request.h"
+#include <event2/buffer.h>
+#include <event2/http.h>
+#include <evhttp.h>
+#include <algorithm>
+#include <utility>
+
+static const char UrlKeyModel[] = "model";
+static const char UrlKeyVersion[] = "version";
+static const char UrlSplit[] = "/";
+static const char UrlKeyEnd[] = ":";
+
+namespace mindspore {
+namespace serving {
+
+DecomposeEvRequest::DecomposeEvRequest(struct evhttp_request *request, int max_msg_size)
+    : event_request_(request), max_msg_size_(max_msg_size) {}
+
+DecomposeEvRequest::~DecomposeEvRequest() {
+  if (event_request_ && evhttp_request_is_owned(event_request_)) {
+    evhttp_request_free(event_request_);
+  }
+}
+
+std::string DecomposeEvRequest::UrlQuery(const std::string &url, const std::string &key) {
+  std::string::size_type start_pos(0);
+  if (key == UrlKeyEnd) {
+    if ((start_pos = url_.find(UrlKeyEnd)) != std::string::npos) {
+      return url_.substr(start_pos + 1, url_.size());
+    }
+  }
+
+  int key_size = key.size() + 1;
+  std::string::size_type end_pos(0);
+  if ((start_pos = url.find(key)) != std::string::npos) {
+    end_pos = std::min(url.find(UrlSplit, start_pos + key_size), url.find(UrlKeyEnd, start_pos + key_size));
+    if (end_pos != std::string::npos) {
+      return url.substr(start_pos + key_size, end_pos - start_pos - key_size);
+    }
+  }
+  return "";
+}
+
+Status DecomposeEvRequest::GetPostMessageToJson() {
+  Status status(SUCCESS);
+  std::string message;
+  size_t input_size = evbuffer_get_length(event_request_->input_buffer);
+  if (input_size == 0) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message invalid");
+    return status;
+  } else if (input_size > max_msg_size_) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message is bigger than " + std::to_string(max_msg_size_));
+    return status;
+  } else {
+    message.resize(input_size);
+    auto src_data = evbuffer_pullup(event_request_->input_buffer, -1);
+    if (src_data == nullptr) {
+      ERROR_INFER_STATUS(status, FAILED, "get http message failed.");
+      return status;
+    }
+    if (memcpy_s(message.data(), input_size, src_data, input_size) != EOK) {
+      ERROR_INFER_STATUS(status, FAILED, "copy http message failed.");
+      return status;
+    }
+  }
+  MSI_LOG_INFO << "get request message: [" << message << "]";
+  MSI_TIME_STAMP_START(ParseJson)
+  try {
+    request_message_ = nlohmann::json::parse(message);
+  } catch (nlohmann::json::exception &e) {
+    std::string json_exception = e.what();
+    std::string error_message = "Illegal JSON format." + json_exception;
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, error_message);
+    return status;
+  }
+  MSI_TIME_STAMP_END(ParseJson)
+
+  return status;
+}
+
+Status DecomposeEvRequest::CheckRequestMethodValid() {
+  Status status(SUCCESS);
+  switch (evhttp_request_get_command(event_request_)) {
+    case EVHTTP_REQ_POST:
+      request_method_ = "POST";
+      return status;
+    default:
+      ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message only support POST right now");
+      return status;
+  }
+}
+
+Status DecomposeEvRequest::Decompose() {
+  Status status(SUCCESS);
+  status = CheckRequestMethodValid();
+  if (status != SUCCESS) {
+    return status;
+  }
+
+  status = GetPostMessageToJson();
+  if (status != SUCCESS) {
+    return status;
+  }
+
+  // eg: /model/resnet/version/1:predict
+  url_ = evhttp_request_get_uri(event_request_);
+  if (url_.empty()) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "evhttp url is empty.");
+    return status;
+  }
+  MSI_LOG_INFO << "url_: " << url_;
+
+  model_name_ = UrlQuery(url_, UrlKeyModel);
+  if (model_name_.empty()) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "please check url, the keyword:[model] must contain.");
+    return status;
+  }
+  MSI_LOG_INFO << "model_name_: " << model_name_;
+  if (url_.find(UrlKeyVersion) != std::string::npos) {
+    auto version_str = UrlQuery(url_, UrlKeyVersion);
+    try {
+      version_ = std::stol(version_str);
+    } catch (const std::invalid_argument &) {
+      ERROR_INFER_STATUS(status, INVALID_INPUTS, "please check url, the keyword:[version] must contain.");
+      return status;
+    } catch (const std::out_of_range &) {
+      ERROR_INFER_STATUS(status, INVALID_INPUTS, "please check url, the keyword:[version] out of range.");
+      return status;
+    }
+    MSI_LOG_INFO << "version_: " << version_;
+  }
+
+  service_method_ = UrlQuery(url_, UrlKeyEnd);
+  if (service_method_.empty()) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "please check url, the keyword:[service method] must contain.");
+    return status;
+  }
+  MSI_LOG_INFO << "service_method_: " << service_method_;
+  return status;
+}
+
+RestfulRequest::RestfulRequest(std::shared_ptr<DecomposeEvRequest> request)
+    : decompose_event_request_(std::move(request)) {}
+
+RestfulRequest::~RestfulRequest() {
+  if (replay_buffer_ != nullptr) {
+    evbuffer_free(replay_buffer_);
+  }
+}
+
+Status RestfulRequest::RestfulReplayBufferInit() {
+  Status status(SUCCESS);
+  replay_buffer_ = evbuffer_new();
+  if (replay_buffer_ == nullptr) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "creat restful replay buffer fail");
+    return status;
+  }
+  return status;
+}
+
+Status RestfulRequest::RestfulReplay(const std::string &replay) {
+  Status status(SUCCESS);
+  if (replay_buffer_ == nullptr) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "replay_buffer_ is nullptr");
+    return status;
+  }
+  if (decompose_event_request_ == nullptr) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "replay_buffer_ is nullptr");
+    return status;
+  }
+  if (decompose_event_request_->event_request_ == nullptr) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "replay_buffer_ is nullptr");
+    return status;
+  }
+  MSI_LOG_INFO << "replay message: " << replay;
+  evbuffer_add(replay_buffer_, replay.data(), replay.size());
+  evhttp_send_reply(decompose_event_request_->event_request_, HTTP_OK, "Client", replay_buffer_);
+  return status;
+}
+
+Status RestfulRequest::ErrorMessage(Status status) {
+  Status error_status(SUCCESS);
+  nlohmann::json error_json = {{"error_message", status.StatusMessage()}};
+  std::string out_error_str = error_json.dump();
+  if ((error_status = RestfulReplay(out_error_str)) != SUCCESS) {
+    return error_status;
+  }
+  return error_status;
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/master/restful/restful_request.h

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_RESTFUL_REQUEST_H
+#define MINDSPORE_RESTFUL_REQUEST_H
+
+#include <event2/event.h>
+#include <event2/http.h>
+#include <string>
+#include <memory>
+#include <nlohmann/json.hpp>
+#include "common/serving_common.h"
+
+namespace mindspore {
+namespace serving {
+
+class DecomposeEvRequest {
+ public:
+  explicit DecomposeEvRequest(struct evhttp_request *request, int max_msg_size);
+  ~DecomposeEvRequest();
+  std::string UrlQuery(const std::string &url, const std::string &key);
+  Status CheckRequestMethodValid();
+  Status Decompose();
+  Status GetPostMessageToJson();
+
+  evhttp_request *event_request_;
+  std::string request_method_;
+  std::string model_name_;
+  std::string url_;
+  std::string service_method_;
+  uint32_t version_{};
+  uint32_t max_msg_size_{};
+  nlohmann::json request_message_;
+};
+
+class RestfulRequest {
+ public:
+  explicit RestfulRequest(std::shared_ptr<DecomposeEvRequest> request);
+  ~RestfulRequest();
+
+  RestfulRequest(const RestfulRequest &other) = delete;
+  RestfulRequest &operator=(const RestfulRequest &other) = delete;
+
+  Status RestfulReplayBufferInit();
+  Status RestfulReplay(const std::string &replay);
+  Status ErrorMessage(Status status);
+  std::shared_ptr<DecomposeEvRequest> decompose_event_request() { return decompose_event_request_; }
+
+ private:
+  std::shared_ptr<DecomposeEvRequest> decompose_event_request_{nullptr};
+  evbuffer *replay_buffer_ = nullptr;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_RESTFUL_REQUEST_H

mindspore_serving/ccsrc/master/restful/restful_server.cc

@@ -0,0 +1,211 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "master/restful/restful_server.h"
+#include <unistd.h>
+#include <memory>
+#include <nlohmann/json.hpp>
+#include "master/restful/http_handle.h"
+namespace mindspore::serving {
+
+int id = 0;
+Status RestfulServer::Run(const std::shared_ptr<RestfulRequest> &restful_request) {
+  Status status(SUCCESS);
+  // MSI_TIME_STAMP_START(DoSometing)
+  // // ke cai start --- sync
+  // ++id;
+  // for (int i = 0; i < 10; ++i) {
+  //   MSI_LOG_INFO << "id:" << id << "--" << i << " ";
+  //   usleep(600);
+  // }
+  // MSI_TIME_STAMP_END(DoSometing)
+  // // -----------------
+
+  // // st error
+  // if (id == 5) {
+  //   ERROR_INFER_STATUS(status, INVALID_INPUTS, "id == 5");
+  //   if ((status = restful_request->ErrorMessage(status)) != SUCCESS) {
+  //     return status;
+  //   }
+  //   return status;
+  // }
+  nlohmann::json predict_json;
+  std::string predict_str;
+  status = HandleRestfulRequest(restful_request, &predict_json);
+  if (status != SUCCESS) {
+    predict_str = status.StatusMessage();
+    nlohmann::json js;
+    js["error_msg"] = predict_str;
+    predict_str = js.dump();
+  } else {
+    predict_str = predict_json.dump();
+  }
+  restful_request->RestfulReplay(predict_str);
+  return status;
+}
+
+void RestfulServer::Committer(const std::shared_ptr<RestfulRequest> &restful_request) {
+  thread_pool_.commit([restful_request, this]() { Run(restful_request); });
+}
+
+void RestfulServer::DispatchEvHttpRequest(evhttp_request *request) {
+  Status status(SUCCESS);
+
+  auto de_request = std::make_unique<DecomposeEvRequest>(request, max_msg_size_);
+  if (de_request == nullptr) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "de_request is nullptr.");
+    return;
+  }
+  Status de_status = de_request->Decompose();
+  auto restful_request = std::make_shared<RestfulRequest>(std::move(de_request));
+  if (restful_request == nullptr) {
+    ERROR_INFER_STATUS(status, INVALID_INPUTS, "restful_request is nullptr.");
+    return;
+  }
+  status = restful_request->RestfulReplayBufferInit();
+  if (status != SUCCESS) {
+    if ((status = restful_request->ErrorMessage(status)) != SUCCESS) {
+      return;
+    }
+    return;
+  }
+
+  if (de_status != SUCCESS) {
+    (void)restful_request->ErrorMessage(de_status);
+    return;
+  }
+  Committer(restful_request);
+}
+
+void RestfulServer::EvCallBack(evhttp_request *request, void *arg) {
+  auto *restful_server = static_cast<RestfulServer *>(arg);
+  restful_server->DispatchEvHttpRequest(const_cast<evhttp_request *>(request));
+}
+
+Status RestfulServer::CreatRestfulServer(int time_out_second) {
+  evthread_use_pthreads();
+  auto free_event_base = [this] {
+    if (event_base_ != nullptr) {
+      event_base_free(event_base_);
+      event_base_ = nullptr;
+    }
+  };
+
+  event_base_ = event_base_new();
+  Status status(SUCCESS);
+  if (event_base_ == nullptr) {
+    status = INFER_STATUS_LOG_ERROR(SYSTEM_ERROR)
+             << "Serving Error: RESTful server start failed, new http event failed";
+    return status;
+  }
+  event_http_ = evhttp_new(event_base_);
+  if (event_http_ == nullptr) {
+    status = INFER_STATUS_LOG_ERROR(SYSTEM_ERROR)
+             << "Serving Error: RESTful server start failed, create http server faild";
+    free_event_base();
+    return status;
+  }
+
+  evhttp_set_gencb(event_http_, &EvCallBack, this);
+  evhttp_set_timeout(event_http_, time_out_second);
+  return SUCCESS;
+}
+
+Status RestfulServer::StartRestfulServer() {
+  auto free_event_base = [this] {
+    if (event_base_ != nullptr) {
+      event_base_free(event_base_);
+      event_base_ = nullptr;
+    }
+  };
+  auto free_evhttp = [this] {
+    if (event_http_ != nullptr) {
+      evhttp_free(event_http_);
+      event_http_ = nullptr;
+    }
+  };
+
+  Status status(SUCCESS);
+  struct sockaddr_in sin = {};
+  sin.sin_family = AF_INET;
+  sin.sin_port = htons(restful_port_);
+  auto listener = evconnlistener_new_bind(event_base_, nullptr, nullptr,
+                                          LEV_OPT_REUSEABLE | LEV_OPT_CLOSE_ON_EXEC | LEV_OPT_CLOSE_ON_FREE, -1,
+                                          reinterpret_cast<struct sockaddr *>(&sin), sizeof(sin));
+
+  if (listener == nullptr) {
+    status = INFER_STATUS_LOG_ERROR(SYSTEM_ERROR)
+             << "Serving Error: RESTful server start failed, create http listener faild, port " << restful_port_;
+    free_event_base();
+    free_evhttp();
+    return status;
+  }
+  auto bound = evhttp_bind_listener(event_http_, listener);
+  if (bound == nullptr) {
+    status = INFER_STATUS_LOG_ERROR(SYSTEM_ERROR)
+             << "Serving Error: RESTful server start failed, bind http listener to server faild, port "
+             << restful_port_;
+    evconnlistener_free(listener);
+    free_event_base();
+    free_evhttp();
+    return status;
+  }
+
+  auto event_http_run = [this]() {
+    MSI_LOG(INFO) << "MS Serving restful listening on " << restful_ip_ << ":" << restful_port_;
+    std::cout << "Serving: MS Serving RESTful start success, listening on " << restful_ip_ << ":" << restful_port_
+              << std::endl;
+    event_base_dispatch(event_base_);
+  };
+  event_thread_ = std::thread(event_http_run);
+  in_running_ = true;
+  return SUCCESS;
+}
+
+Status RestfulServer::Start(const std::string &ip, uint32_t restful_port, int max_msg_size, int time_out_second) {
+  Status status(SUCCESS);
+
+  restful_ip_ = ip;
+  restful_port_ = restful_port;
+  max_msg_size_ = static_cast<int>(max_msg_size * (uint32_t(1) << 20));
+
+  status = CreatRestfulServer(time_out_second);
+  if (status != SUCCESS) {
+    return status;
+  }
+  status = StartRestfulServer();
+  if (status != SUCCESS) {
+    return status;
+  }
+  return status;
+}
+
+void RestfulServer::Stop() {
+  if (in_running_) {
+    event_base_loopexit(event_base_, nullptr);
+    event_thread_.join();
+  }
+  if (event_base_ != nullptr) {
+    event_base_free(event_base_);
+    event_base_ = nullptr;
+  }
+  if (event_http_ != nullptr) {
+    evhttp_free(event_http_);
+    event_http_ = nullptr;
+  }
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/master/restful/restful_server.h

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_RESTFUL_SERVER_H
+#define MINDSPORE_RESTFUL_SERVER_H
+
+#include <event.h>
+#include <event2/event.h>
+#include <event2/http.h>
+#include <event2/listener.h>
+#include <event2/thread.h>
+#include <evhttp.h>
+#include <memory>
+#include <future>
+#include <string>
+#include <utility>
+
+#include "master/restful/restful_request.h"
+#include "common/serving_common.h"
+#include "common/thread_pool.h"
+
+namespace mindspore::serving {
+
+constexpr const uint32_t kDefaultRestfulThreadPoolNum = 20;
+
+typedef void (*RestfulFunc)(struct evhttp_request *const req, void *const arg);
+
+class MS_API RestfulServer {
+ public:
+  RestfulServer() : thread_pool_(kDefaultRestfulThreadPoolNum) {}
+  ~RestfulServer() { Stop(); }
+
+  Status Start(const std::string &ip, uint32_t restful_port, int max_msg_size, int time_out_second);
+  void Stop();
+
+ private:
+  Status CreatRestfulServer(int time_out_second);
+  static void EvCallBack(evhttp_request *request, void *arg);
+  void DispatchEvHttpRequest(evhttp_request *request);
+  void Committer(const std::shared_ptr<RestfulRequest> &restful_request);
+  Status Run(const std::shared_ptr<RestfulRequest> &restful_request);
+  Status StartRestfulServer();
+
+  std::string restful_ip_;
+  uint32_t restful_port_{};
+  int max_msg_size_{};
+  bool in_running_ = false;
+
+  struct evhttp *event_http_ = nullptr;
+  struct event_base *event_base_ = nullptr;
+  std::thread event_thread_;
+  ThreadPool thread_pool_;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_RESTFUL_SERVER_H

mindspore_serving/ccsrc/master/server.cc

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "master/server.h"
+
+#include <atomic>
+#include <future>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/proto_tensor.h"
+#include "common/serving_common.h"
+#include "common/exit_handle.h"
+#include "master/dispacther.h"
+#include "master/grpc/grpc_process.h"
+#include "master/restful/http_process.h"
+#include "worker/context.h"
+
+namespace mindspore {
+namespace serving {
+
+Status Server::StartGrpcServer(const std::string &ip, uint32_t grpc_port, int max_msg_mb_size) {
+  if (max_msg_mb_size > gRpcMaxMBMsgSize) {
+    MSI_LOG_WARNING << "The maximum Serving gRPC message size is 512MB and will be updated from " << max_msg_mb_size
+                    << "MB to 512MB";
+    max_msg_mb_size = gRpcMaxMBMsgSize;
+  }
+  return grpc_server_.Start(std::make_shared<MSServiceImpl>(dispatcher_), ip, grpc_port, max_msg_mb_size,
+                            "Serving gRPC");
+}
+
+Status Server::StartGrpcMasterServer(const std::string &ip, uint32_t grpc_port) {
+  return grpc_manager_server_.Start(std::make_shared<MSMasterImpl>(dispatcher_), ip, grpc_port, -1, "Master gRPC");
+}
+
+Status Server::StartRestfulServer(const std::string &ip, uint32_t restful_port, int max_msg_mb_size,
+                                  int time_out_second) {
+  return restful_server_.Start(ip, restful_port, max_msg_mb_size, time_out_second);
+  // restful_server.Start(http_handler_msg, ip, restful_port, max_msg_mb_size, time_out_second);
+}
+
+void Server::Clear() { dispatcher_->Clear(); }
+
+Server::Server() = default;
+
+Server &Server::Instance() {
+  static Server server;
+  ExitHandle::Instance().InitSignalHandle();
+  return server;
+}
+
+Server::~Server() = default;
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/master/server.h

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_SERVER_H
+#define MINDSPORE_SERVER_H
+
+#include <memory>
+#include <string>
+#include "common/serving_common.h"
+#include "common/grpc_server.h"
+#include "master/restful/restful_server.h"
+#include "master/dispacther.h"
+
+namespace mindspore {
+namespace serving {
+
+class MS_API Server {
+ public:
+  Server();
+  ~Server();
+  Status StartGrpcServer(const std::string &ip, uint32_t grpc_port, int max_msg_mb_size = gRpcDefaultMsgMBSize);
+  Status StartGrpcMasterServer(const std::string &ip, uint32_t grpc_port);
+  Status StartRestfulServer(const std::string &ip, uint32_t restful_port, int max_msg_mb_size = gRpcDefaultMsgMBSize,
+                            int time_out_second = 100);
+  void Clear();
+
+  std::shared_ptr<Dispatcher> GetDispatcher() { return dispatcher_; }
+
+  static Server &Instance();
+
+ private:
+  GrpcServer grpc_server_;
+  GrpcServer grpc_manager_server_;
+  RestfulServer restful_server_;
+  std::shared_ptr<Dispatcher> dispatcher_ = std::make_shared<Dispatcher>();
+};
+}  // namespace serving
+}  // namespace mindspore
+#endif  // MINDSPORE_SERVER_H

mindspore_serving/ccsrc/python/master/master_py.cc

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "python/master/master_py.h"
+#include "common/exit_handle.h"
+#include "worker/task_queue.h"
+
+namespace mindspore::serving {
+
+void PyMaster::StartGrpcServer(const std::string &ip, uint32_t grpc_port, int max_msg_mb_size) {
+  auto status = Server::Instance().StartGrpcServer(ip, grpc_port, max_msg_mb_size);
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+}
+
+void PyMaster::StartGrpcMasterServer(const std::string &ip, uint32_t grpc_port) {
+  auto status = Server::Instance().StartGrpcMasterServer(ip, grpc_port);
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+}
+
+void PyMaster::StartRestfulServer(const std::string &ip, uint32_t grpc_port, int max_msg_mb_size) {
+  auto status = Server::Instance().StartRestfulServer(ip, grpc_port, max_msg_mb_size);
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+}
+
+void PyMaster::WaitAndClear() {
+  ExitHandle::Instance().MasterWait();
+  Server::Instance().Clear();
+  MSI_LOG_INFO << "Python server end wait and clear";
+}
+
+void PyMaster::Stop() { ExitHandle::Instance().Stop(); }
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/python/master/master_py.h

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVER_PY_H
+#define MINDSPORE_SERVER_PY_H
+
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
+#include <pybind11/stl.h>
+#include <string>
+#include <memory>
+#include "common/serving_common.h"
+#include "master/grpc/grpc_process.h"
+#include "common/grpc_server.h"
+#include "master/restful/restful_server.h"
+#include "master/server.h"
+#include "master/dispacther.h"
+
+namespace py = pybind11;
+
+namespace mindspore {
+namespace serving {
+
+class MS_API PyMaster {
+ public:
+  static void StartGrpcServer(const std::string &ip, uint32_t grpc_port, int max_msg_mb_size = 100);
+  static void StartGrpcMasterServer(const std::string &ip, uint32_t grpc_port);
+  static void StartRestfulServer(const std::string &ip, uint32_t grpc_port, int max_msg_mb_size = 100);
+  static void WaitAndClear();
+  static void Stop();
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVER_PY_H

mindspore_serving/ccsrc/python/serving_py.cc

@@ -0,0 +1,146 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+#include "python/worker/preprocess_py.h"
+#include "python/worker/postprocess_py.h"
+#include "python/worker/worker_py.h"
+#include "python/tensor_py.h"
+#include "common/servable.h"
+#include "worker/context.h"
+#include "python/master/master_py.h"
+#include "master/dispacther.h"
+
+namespace mindspore::serving {
+
+PYBIND11_MODULE(_mindspore_serving, m) {
+  py::class_<PyPreprocessStorage, std::shared_ptr<PyPreprocessStorage>>(m, "PreprocessStorage_")
+    .def(py::init<>())
+    .def_static("get_instance", &PyPreprocessStorage::Instance)
+    .def("register", &PyPreprocessStorage::Register)
+    .def("get_pycpp_preprocess_info", &PyPreprocessStorage::GetPyCppPreprocessInfo);
+
+  py::class_<PyPostprocessStorage, std::shared_ptr<PyPostprocessStorage>>(m, "PostprocessStorage_")
+    .def(py::init<>())
+    .def_static("get_instance", &PyPostprocessStorage::Instance)
+    .def("register", &PyPostprocessStorage::Register)
+    .def("get_pycpp_postprocess_info", &PyPostprocessStorage::GetPyCppPostprocessInfo);
+
+  py::enum_<PredictPhaseTag>(m, "PredictPhaseTag_")
+    .value("kPredictPhaseTag_Input", PredictPhaseTag::kPredictPhaseTag_Input)
+    .value("kPredictPhaseTag_Preproces", PredictPhaseTag::kPredictPhaseTag_Preproces)
+    .value("kPredictPhaseTag_Predict", PredictPhaseTag::kPredictPhaseTag_Predict)
+    .value("kPredictPhaseTag_Postprocess", PredictPhaseTag::kPredictPhaseTag_Postprocess)
+    .export_values();
+
+  py::class_<MethodSignature>(m, "MethodSignature_")
+    .def(py::init<>())
+    .def_readwrite("method_name", &MethodSignature::method_name)
+    .def_readwrite("inputs", &MethodSignature::inputs)
+    .def_readwrite("outputs", &MethodSignature::outputs)
+    .def_readwrite("preprocess_name", &MethodSignature::preprocess_name)
+    .def_readwrite("preprocess_inputs", &MethodSignature::preprocess_inputs)
+    .def_readwrite("postprocess_name", &MethodSignature::postprocess_name)
+    .def_readwrite("postprocess_inputs", &MethodSignature::postprocess_inputs)
+    .def_readwrite("servable_name", &MethodSignature::servable_name)
+    .def_readwrite("servable_inputs", &MethodSignature::servable_inputs)
+    .def_readwrite("returns", &MethodSignature::returns);
+
+  py::class_<RequestSpec>(m, "RequestSpec_")
+    .def(py::init<>())
+    .def_readwrite("servable_name", &RequestSpec::servable_name)
+    .def_readwrite("version_number", &RequestSpec::version_number)
+    .def_readwrite("method_name", &RequestSpec::method_name);
+
+  py::class_<ServableMeta>(m, "ServableMeta_")
+    .def(py::init<>())
+    .def_readwrite("servable_name", &ServableMeta::servable_name)
+    .def_readwrite("inputs_count", &ServableMeta::inputs_count)
+    .def_readwrite("outputs_count", &ServableMeta::outputs_count)
+    .def_readwrite("servable_file", &ServableMeta::servable_file)
+    .def_readwrite("with_batch_dim", &ServableMeta::with_batch_dim)
+    .def("set_model_format", &ServableMeta::SetModelFormat);
+
+  py::class_<ServableSignature>(m, "ServableSignature_")
+    .def(py::init<>())
+    .def_readwrite("servable_meta", &ServableSignature::servable_meta)
+    .def_readwrite("methods", &ServableSignature::methods);
+
+  py::class_<ServableStorage, std::shared_ptr<ServableStorage>>(m, "ServableStorage_")
+    .def(py::init<>())
+    .def_static("get_instance", &ServableStorage::Instance)
+    .def("register_servable", &ServableStorage::Register)
+    .def("register_servable_input_output_info", &ServableStorage::RegisterInputOutputInfo)
+    .def("get_servable_input_output_info", &ServableStorage::GetInputOutputInfo)
+    .def("register_method", &ServableStorage::RegisterMethod)
+    .def("declare_servable", &ServableStorage::DeclareServable);
+
+  py::class_<TaskContext>(m, "TaskContext_").def(py::init<>());
+
+  py::class_<TaskItem>(m, "TaskItem_")
+    .def(py::init<>())
+    .def_readwrite("task_type", &TaskItem::task_type)
+    .def_readwrite("name", &TaskItem::name)
+    .def_property_readonly("instance_list",
+                           [](const TaskItem &item) {
+                             py::tuple instances(item.instance_list.size());
+                             for (size_t i = 0; i < item.instance_list.size(); i++) {
+                               instances[i] = PyTensor::AsNumpyTuple(item.instance_list[i].data);
+                             }
+                             return instances;
+                           })
+    .def_readwrite("context_list", &TaskItem::context_list);
+
+  py::class_<PyWorker>(m, "Worker_")
+    .def_static("start_servable", &PyWorker::StartServable)
+    .def_static("start_servable_in_master", &PyWorker::StartServableInMaster)
+    .def_static("get_batch_size", &PyWorker::GetBatchSize)
+    .def_static("wait_and_clear", &PyWorker::WaitAndClear, py::call_guard<py::gil_scoped_release>())
+    .def_static("stop", PyWorker::Stop)
+    .def_static("get_py_task", &PyWorker::GetPyTask, py::call_guard<py::gil_scoped_release>())
+    .def_static("try_get_preprocess_py_task", &PyWorker::TryGetPreprocessPyTask)
+    .def_static("try_get_postprocess_py_task", &PyWorker::TryGetPostprocessPyTask)
+    .def_static("push_preprocess_result", &PyWorker::PushPreprocessPyResult)
+    .def_static("push_preprocess_failed", &PyWorker::PushPreprocessPyFailed)
+    .def_static("push_postprocess_result", &PyWorker::PushPostprocessPyResult)
+    .def_static("push_postprocess_failed", &PyWorker::PushPostprocessPyFailed);
+
+  py::class_<ServableContext, std::shared_ptr<ServableContext>>(m, "Context_")
+    .def(py::init<>())
+    .def_static("get_instance", &ServableContext::Instance)
+    .def("set_device_type_str",
+         [](ServableContext &context, const std::string &device_type) {
+           auto status = context.SetDeviceTypeStr(device_type);
+           if (status != SUCCESS) {
+             MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+           }
+         })
+    .def("set_device_id", &ServableContext::SetDeviceId);
+
+  py::class_<PyMaster, std::shared_ptr<PyMaster>>(m, "Master_")
+    .def_static("start_grpc_server", &PyMaster::StartGrpcServer)
+    .def_static("start_grpc_master_server", &PyMaster::StartGrpcMasterServer)
+    .def_static("start_restful_server", &PyMaster::StartRestfulServer)
+    .def_static("wait_and_clear", &PyMaster::WaitAndClear, py::call_guard<py::gil_scoped_release>())
+    .def_static("stop", &PyMaster::Stop);
+
+  (void)py::module::import("atexit").attr("register")(py::cpp_function{[&]() -> void {
+    Server::Instance().Clear();
+    Worker::GetInstance().Clear();
+  }});
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/python/tensor_py.cc

@@ -0,0 +1,259 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "python/tensor_py.h"
+#include <pybind11/pytypes.h>
+#include <string>
+#include <vector>
+#include <memory>
+#include "mindspore_serving/ccsrc/common/tensor.h"
+
+namespace mindspore::serving {
+
+static std::vector<ssize_t> GetStrides(const std::vector<ssize_t> &shape, ssize_t item_size) {
+  std::vector<ssize_t> strides;
+  strides.reserve(shape.size());
+  const auto ndim = shape.size();
+  for (size_t i = 0; i < ndim; ++i) {
+    auto stride = item_size;
+    for (size_t j = i + 1; j < ndim; ++j) {
+      stride *= shape[j];
+    }
+    strides.push_back(stride);
+  }
+  return strides;
+}
+
+DataType NumpyTensor::GetDataType(const py::buffer_info &buf) {
+  if (buf.format.size() == 1) {
+    switch (buf.format.front()) {
+      case 'e':
+      case 'f':
+      case 'd':
+        switch (buf.itemsize) {
+          case 2:
+            return kMSI_Float16;
+          case 4:
+            return kMSI_Float32;
+          case 8:
+            return kMSI_Float64;
+        }
+        break;
+      case 'b':
+      case 'h':
+      case 'i':
+      case 'l':
+      case 'q':
+        switch (buf.itemsize) {
+          case 1:
+            return kMSI_Int8;
+          case 2:
+            return kMSI_Int16;
+          case 4:
+            return kMSI_Int32;
+          case 8:
+            return kMSI_Int64;
+        }
+        break;
+      case 'B':
+      case 'H':
+      case 'I':
+      case 'L':
+      case 'Q':
+        switch (buf.itemsize) {
+          case 1:
+            return kMSI_Uint8;
+          case 2:
+            return kMSI_Uint16;
+          case 4:
+            return kMSI_Uint32;
+          case 8:
+            return kMSI_Uint64;
+        }
+        break;
+      case '?':
+        return kMSI_Bool;
+    }
+  }
+  MSI_LOG(WARNING) << "Unsupported DataType format " << buf.format << " item size " << buf.itemsize;
+  return kMSI_Unknown;
+}
+
+static std::string GetPyTypeFormat(DataType data_type) {
+  switch (data_type) {
+    case kMSI_Float16:
+      return "e";
+    case kMSI_Float32:
+      return py::format_descriptor<float>::format();
+    case kMSI_Float64:
+      return py::format_descriptor<double>::format();
+    case kMSI_Uint8:
+      return py::format_descriptor<uint8_t>::format();
+    case kMSI_Uint16:
+      return py::format_descriptor<uint16_t>::format();
+    case kMSI_Uint32:
+      return py::format_descriptor<uint32_t>::format();
+    case kMSI_Uint64:
+      return py::format_descriptor<uint64_t>::format();
+    case kMSI_Int8:
+      return py::format_descriptor<int8_t>::format();
+    case kMSI_Int16:
+      return py::format_descriptor<int16_t>::format();
+    case kMSI_Int32:
+      return py::format_descriptor<int32_t>::format();
+    case kMSI_Int64:
+      return py::format_descriptor<int64_t>::format();
+    case kMSI_Bool:
+      return py::format_descriptor<bool>::format();
+    default:
+      MSI_LOG(WARNING) << "Unsupported DataType " << data_type << ".";
+      return "";
+  }
+}
+
+static bool IsCContiguous(const py::array &input) {
+  auto flags = static_cast<unsigned int>(input.flags());
+  return (flags & pybind11::detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_) != 0;
+}
+
+TensorBasePtr PyTensor::MakeTensor(const py::array &input) {
+  // Get input buffer info.
+  py::buffer_info buf = input.request();
+  // Check data types.
+  auto buf_type = NumpyTensor::GetDataType(buf);
+  if (buf_type == kMSI_Unknown) {
+    MSI_LOG(EXCEPTION) << "Unsupported tensor type!";
+  }
+  // Convert input array to C contiguous if need.
+  std::unique_ptr<char[]> tmp_buf;
+  if (!IsCContiguous(input)) {
+    Py_buffer pybuf;
+    if (PyObject_GetBuffer(input.ptr(), &pybuf, PyBUF_ANY_CONTIGUOUS)) {
+      MSI_LOG(EXCEPTION) << "Failed to get buffer from the input!";
+    }
+    tmp_buf = std::make_unique<char[]>(pybuf.len);
+    if (PyBuffer_ToContiguous(tmp_buf.get(), &pybuf, pybuf.len, 'C')) {
+      MSI_LOG(EXCEPTION) << "Can't copy numpy.ndarray to a contiguous buffer.";
+    }
+    PyBuffer_Release(&pybuf);
+    buf.ptr = tmp_buf.get();
+  }
+  // Get tensor shape.
+  std::vector<int64_t> shape(buf.shape.begin(), buf.shape.end());
+  return std::make_shared<Tensor>(buf_type, shape, buf.ptr, buf.size * buf.itemsize);
+}
+
+/// Creates a Tensor from a numpy array without copy
+TensorBasePtr PyTensor::MakeTensorNoCopy(const py::array &input) {
+  // Check format.
+  if (!IsCContiguous(input)) {
+    MSI_LOG(EXCEPTION) << "Array should be C contiguous.";
+  }
+  // Get input buffer info.
+  py::buffer_info buf = input.request();
+  // Get tensor dtype and check it.
+  auto dtype = NumpyTensor::GetDataType(buf);
+  if (dtype == kMSI_Unknown) {
+    MSI_LOG(EXCEPTION) << "Unsupported data type!";
+  }
+  // Make a tensor with shared data with numpy array.
+  auto tensor_data = std::make_shared<NumpyTensor>(std::move(buf));
+  return tensor_data;
+}
+
+py::array PyTensor::AsNumpy(TensorBasePtr tensor, bool copy) {
+  auto data_numpy = std::dynamic_pointer_cast<NumpyTensor>(tensor);
+  if (data_numpy) {
+    return data_numpy->py_array();
+  }
+  if (tensor->is_bytes_val_data()) {
+    if (tensor->bytes_data_size() != 1) {
+      return py::array();
+    }
+    const uint8_t *data = nullptr;
+    size_t bytes_len = 0;
+    tensor->get_bytes_data(0, data, bytes_len);
+    std::vector<ssize_t> shape{static_cast<long>(bytes_len)};
+    std::vector<ssize_t> strides = GetStrides(shape, sizeof(uint8_t));
+    py::buffer_info info(reinterpret_cast<void *>(const_cast<uint8_t *>(data)), sizeof(uint8_t),
+                         py::format_descriptor<uint8_t>::format(), 1, shape, strides);
+    if (!copy) {
+      py::array self;
+      return py::array(py::dtype(info), info.shape, info.strides, info.ptr, self);
+    } else {
+      return py::array(py::dtype(info), info.shape, info.strides, info.ptr);
+    }
+  } else {
+    const auto &tensor_shape = tensor->shape();
+    std::vector<ssize_t> shape(tensor_shape.begin(), tensor_shape.end());
+    std::vector<ssize_t> strides = GetStrides(shape, tensor->itemsize());
+    py::buffer_info info(reinterpret_cast<void *>(const_cast<uint8_t *>(tensor->data())),
+                         static_cast<ssize_t>(tensor->itemsize()), GetPyTypeFormat(tensor->data_type()),
+                         static_cast<ssize_t>(tensor_shape.size()), shape, strides);
+
+    if (!copy) {
+      py::array self;
+      return py::array(py::dtype(info), info.shape, info.strides, info.ptr, self);
+    } else {
+      return py::array(py::dtype(info), info.shape, info.strides, info.ptr);
+    }
+  }
+}
+
+py::tuple PyTensor::AsNumpyTuple(const InstanceData &instance_data) {
+  py::tuple numpy_inputs_tuple(instance_data.size());
+  for (size_t i = 0; i < instance_data.size(); i++) {  // inputs
+    numpy_inputs_tuple[i] = PyTensor::AsNumpy(instance_data[i]);
+  }
+  return numpy_inputs_tuple;
+}
+
+InstanceData PyTensor::AsInstanceData(const py::tuple &tuple) {
+  InstanceData instance_data;
+  for (auto &item : tuple) {
+    TensorBasePtr tensor = nullptr;
+    if (py::isinstance<py::str>(item)) {
+      tensor = std::make_shared<Tensor>();
+      tensor->set_data_type(serving::kMSI_String);
+      auto val = item.cast<std::string>();
+      tensor->add_bytes_data(reinterpret_cast<const uint8_t *>(val.data()), val.length());
+    } else if (py::isinstance<py::bytes>(item)) {
+      tensor = std::make_shared<Tensor>();
+      tensor->set_data_type(serving::kMSI_Bytes);
+      auto val = std::string(item.cast<py::bytes>());
+      tensor->add_bytes_data(reinterpret_cast<const uint8_t *>(val.data()), val.length());
+    } else if (py::isinstance<py::bool_>(item)) {
+      auto val = item.cast<bool>();
+      tensor = std::make_shared<Tensor>(serving::kMSI_Bool, std::vector<int64_t>(), &val, sizeof(val));
+    } else if (py::isinstance<py::int_>(item)) {
+      auto val = item.cast<int32_t>();
+      tensor = std::make_shared<Tensor>(serving::kMSI_Int32, std::vector<int64_t>(), &val, sizeof(val));
+    } else if (py::isinstance<py::float_>(item)) {
+      auto val = item.cast<float>();
+      tensor = std::make_shared<Tensor>(serving::kMSI_Float32, std::vector<int64_t>(), &val, sizeof(val));
+    } else {
+      try {
+        tensor = PyTensor::MakeTensorNoCopy(py::cast<py::array>(item));
+      } catch (const std::runtime_error &error) {
+        MSI_LOG_EXCEPTION << "Get illegal result data with type " << py::str(item.get_type()).cast<std::string>();
+      }
+    }
+    instance_data.push_back(tensor);
+  }
+  return instance_data;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/python/tensor_py.h

@@ -0,0 +1,98 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_SERVING_PY_H
+#define MINDSPORE_SERVING_SERVING_PY_H
+
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
+#include <pybind11/stl.h>
+#include <utility>
+#include <vector>
+#include "common/serving_common.h"
+#include "common/instance.h"
+
+namespace py = pybind11;
+
+namespace mindspore::serving {
+
+class NumpyTensor : public TensorBase {
+ public:
+  explicit NumpyTensor(py::buffer_info &&buffer) : buffer_(std::move(buffer)) {}
+  ~NumpyTensor() {
+    py::gil_scoped_acquire acquire;
+    { buffer_ = py::buffer_info(); }
+  }
+  /// py::array object.
+  py::array py_array() const {
+    // Use dummy owner to avoid copy data.
+    py::str dummyOwner;
+    return py::array(py::dtype(buffer_), buffer_.shape, buffer_.strides, buffer_.ptr, dummyOwner);
+  }
+
+  void set_data_type(DataType type) override {
+    MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke set_data_type";
+  }
+  DataType data_type() const override { return GetDataType(buffer_); }
+
+  void set_shape(const std::vector<int64_t> &shape) override {
+    MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke set_shape";
+  }
+  std::vector<int64_t> shape() const override { return buffer_.shape; }
+
+  const uint8_t *data() const override { return static_cast<const uint8_t *>(buffer_.ptr); }
+  size_t data_size() const override { return buffer_.size * buffer_.itemsize; }
+
+  bool resize_data(size_t data_len) override {
+    MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke resize_data";
+    return false;
+  }
+  uint8_t *mutable_data() override {
+    MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke mutable_data";
+    return nullptr;
+  }
+
+  void clear_bytes_data() override { MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke clear_bytes_data"; }
+  void add_bytes_data(const uint8_t *data, size_t bytes_len) override {
+    MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke add_bytes_data";
+  }
+
+  size_t bytes_data_size() const override { return 0; }
+  void get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const override {
+    MSI_LOG_EXCEPTION << "NumpyTensor is readyonly, cannot invoke get_bytes_data";
+  }
+
+  static DataType GetDataType(const py::buffer_info &buf);
+
+ private:
+  py::buffer_info buffer_;
+};
+
+class PyTensor {
+ public:
+  // For all type, but for BYTES type, there can only be one item in bytes_val.
+  // If the tensor data is destroyed when the numpy array is return to python env, the tensor data need to be copied
+  static py::array AsNumpy(TensorBasePtr tensor, bool copy = false);
+  static TensorBasePtr MakeTensor(const py::array &input);
+  static TensorBasePtr MakeTensorNoCopy(const py::array &input);
+
+  static py::tuple AsNumpyTuple(const InstanceData &instance);
+  static InstanceData AsInstanceData(const py::tuple &tuple);
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_SERVING_PY_H

mindspore_serving/ccsrc/python/worker/postprocess_py.cc

@@ -0,0 +1,82 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "python/worker/postprocess_py.h"
+#include <pybind11/embed.h>
+
+namespace mindspore::serving {
+
+size_t PyPostprocess::GetInputsCount(const std::string &postprocess_name) const {
+  size_t inputs_count;
+  size_t outputs_count;
+  (void)PyPostprocessStorage::Instance()->GetPyPostprocessInfo(postprocess_name, inputs_count, outputs_count);
+  return inputs_count;
+}
+
+size_t PyPostprocess::GetOutputsCount(const std::string &postprocess_name) const {
+  size_t inputs_count;
+  size_t outputs_count;
+  (void)PyPostprocessStorage::Instance()->GetPyPostprocessInfo(postprocess_name, inputs_count, outputs_count);
+  return outputs_count;
+}
+
+std::shared_ptr<PyPostprocessStorage> PyPostprocessStorage::Instance() {
+  static std::shared_ptr<PyPostprocessStorage> instance;
+  if (instance == nullptr) {
+    instance = std::make_shared<PyPostprocessStorage>();
+  }
+  return instance;
+}
+
+void PyPostprocessStorage::Register(const std::string &postprocess_name, size_t inputs_count, size_t outputs_count) {
+  postprocess_infos_[postprocess_name] = std::make_pair(inputs_count, outputs_count);
+  MSI_LOG_INFO << "Register python postprocess " << postprocess_name;
+  PostprocessStorage::Instance().Register(postprocess_name, py_postprocess_);
+}
+
+bool PyPostprocessStorage::GetPyPostprocessInfo(const std::string &postprocess_name, size_t &inputs_count,
+                                                size_t &outputs_count) {
+  auto it = postprocess_infos_.find(postprocess_name);
+  if (it == postprocess_infos_.end()) {
+    return false;
+  }
+  inputs_count = it->second.first;
+  outputs_count = it->second.second;
+  return true;
+}
+
+std::vector<size_t> PyPostprocessStorage::GetPyCppPostprocessInfo(const std::string &postprocess_name) const {
+  std::vector<size_t> results;
+  auto postprocess = PostprocessStorage::Instance().GetPostprocess(postprocess_name);
+  if (!postprocess) {
+    return results;
+  }
+  size_t inputs_count = postprocess->GetInputsCount(postprocess_name);
+  size_t outputs_count = postprocess->GetOutputsCount(postprocess_name);
+  if (inputs_count == 0 || outputs_count == 0) {
+    MSI_LOG_ERROR << "Postprocess " + postprocess_name + " input or output names cannot be null";
+    return results;
+  }
+  results.push_back(inputs_count);
+  results.push_back(outputs_count);
+  return results;
+}
+
+PyPostprocessStorage::PyPostprocessStorage() {}
+
+PyPostprocessStorage::~PyPostprocessStorage() = default;
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/python/worker/postprocess_py.h

@@ -0,0 +1,64 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_POSTPROCESS_PY_H
+#define MINDSPORE_SERVING_POSTPROCESS_PY_H
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include <string>
+#include <utility>
+#include "pybind11/pybind11.h"
+#include "pybind11/numpy.h"
+#include "pybind11/stl.h"
+#include "worker/postprocess.h"
+#include "python/tensor_py.h"
+
+namespace py = pybind11;
+
+namespace mindspore::serving {
+
+class PyPostprocess : public PostprocessBase {
+ public:
+  Status Postprocess(const std::string &postprocess_name, const InstanceData &input, InstanceData &output) override {
+    MSI_LOG_EXCEPTION << "PyPostprocess::Postprocess not expected to be invoked, preprocess name " << postprocess_name;
+  }
+  size_t GetInputsCount(const std::string &postprocess_name) const override;
+  size_t GetOutputsCount(const std::string &postprocess_name) const override;
+  bool IsPythonPostprocess() const override { return true; }
+};
+
+class MS_API PyPostprocessStorage {
+ public:
+  static std::shared_ptr<PyPostprocessStorage> Instance();
+
+  void Register(const std::string &preprocess_name, size_t inputs_count, size_t outputs_count);
+
+  bool GetPyPostprocessInfo(const std::string &postprocess_name, size_t &inputs_count, size_t &outputs_count);
+
+  std::vector<size_t> GetPyCppPostprocessInfo(const std::string &postprocess_name) const;
+  PyPostprocessStorage();
+  ~PyPostprocessStorage();
+
+ private:
+  std::unordered_map<std::string, std::pair<size_t, size_t>> postprocess_infos_;
+  std::shared_ptr<PyPostprocess> py_postprocess_ = std::make_shared<PyPostprocess>();
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_POSTPROCESS_PY_H

mindspore_serving/ccsrc/python/worker/preprocess_py.cc

@@ -0,0 +1,86 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "python/worker/preprocess_py.h"
+#include <pybind11/embed.h>
+#include <memory>
+#include <vector>
+#include <string>
+#include <utility>
+
+namespace mindspore::serving {
+
+size_t PyPreprocess::GetInputsCount(const std::string &preprocess_name) const {
+  size_t inputs_count;
+  size_t outputs_count;
+  (void)PyPreprocessStorage::Instance()->GetPyPreprocessInfo(preprocess_name, inputs_count, outputs_count);
+  return inputs_count;
+}
+
+size_t PyPreprocess::GetOutputsCount(const std::string &preprocess_name) const {
+  size_t inputs_count;
+  size_t outputs_count;
+  (void)PyPreprocessStorage::Instance()->GetPyPreprocessInfo(preprocess_name, inputs_count, outputs_count);
+  return outputs_count;
+}
+
+std::shared_ptr<PyPreprocessStorage> PyPreprocessStorage::Instance() {
+  static std::shared_ptr<PyPreprocessStorage> instance;
+  if (instance == nullptr) {
+    instance = std::make_shared<PyPreprocessStorage>();
+  }
+  return instance;
+}
+
+void PyPreprocessStorage::Register(const std::string &preprocess_name, size_t inputs_count, size_t outputs_count) {
+  preprocess_infos_[preprocess_name] = std::make_pair(inputs_count, outputs_count);
+  MSI_LOG_INFO << "Register python preprocess " << preprocess_name;
+  PreprocessStorage::Instance().Register(preprocess_name, py_preprocess_);
+}
+
+bool PyPreprocessStorage::GetPyPreprocessInfo(const std::string &preprocess_name, size_t &inputs_count,
+                                              size_t &outputs_count) {
+  auto it = preprocess_infos_.find(preprocess_name);
+  if (it == preprocess_infos_.end()) {
+    return false;
+  }
+  inputs_count = it->second.first;
+  outputs_count = it->second.second;
+  return true;
+}
+
+std::vector<size_t> PyPreprocessStorage::GetPyCppPreprocessInfo(const std::string &preprocess_name) const {
+  std::vector<size_t> results;
+  auto preprocess = PreprocessStorage::Instance().GetPreprocess(preprocess_name);
+  if (!preprocess) {
+    return results;
+  }
+  size_t inputs_count = preprocess->GetInputsCount(preprocess_name);
+  size_t outputs_count = preprocess->GetOutputsCount(preprocess_name);
+  if (inputs_count == 0 || outputs_count == 0) {
+    MSI_LOG_ERROR << "Preprocess " + preprocess_name + " inputs or outputs count cannot be 0";
+    return results;
+  }
+  results.push_back(inputs_count);
+  results.push_back(outputs_count);
+  return results;
+}
+
+PyPreprocessStorage::PyPreprocessStorage() {}
+
+PyPreprocessStorage::~PyPreprocessStorage() = default;
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/python/worker/preprocess_py.h

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_PREPROCESS_PY_H
+#define MINDSPORE_SERVING_PREPROCESS_PY_H
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include <string>
+#include <utility>
+#include "pybind11/pybind11.h"
+#include "pybind11/numpy.h"
+#include "pybind11/stl.h"
+#include "worker/preprocess.h"
+#include "python/tensor_py.h"
+
+namespace py = pybind11;
+
+namespace mindspore::serving {
+
+class PyPreprocess : public PreprocessBase {
+ public:
+  Status Preprocess(const std::string &preprocess_name, const InstanceData &input, InstanceData &output) override {
+    MSI_LOG_EXCEPTION << "PyPreprocess::Preprocess not expected to be invoked, preprocess name " << preprocess_name;
+  }
+  size_t GetInputsCount(const std::string &preprocess_name) const override;
+  size_t GetOutputsCount(const std::string &preprocess_name) const override;
+
+  bool IsPythonPreprocess() const override { return true; }
+};
+
+class MS_API PyPreprocessStorage {
+ public:
+  static std::shared_ptr<PyPreprocessStorage> Instance();
+
+  void Register(const std::string &preprocess_name, size_t inputs_count, size_t outputs_count);
+
+  bool GetPyPreprocessInfo(const std::string &preprocess_name, size_t &inputs_count, size_t &outputs_count);
+
+  std::vector<size_t> GetPyCppPreprocessInfo(const std::string &preprocess_name) const;
+
+  PyPreprocessStorage();
+  ~PyPreprocessStorage();
+
+ private:
+  std::unordered_map<std::string, std::pair<size_t, size_t>> preprocess_infos_;
+  std::shared_ptr<PyPreprocess> py_preprocess_ = std::make_shared<PyPreprocess>();
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_PREPROCESS_PY_H

mindspore_serving/ccsrc/python/worker/worker_py.cc

@@ -0,0 +1,129 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "python/worker/worker_py.h"
+#include <unordered_map>
+#include <vector>
+#include <string>
+#include "common/exit_handle.h"
+#include "worker/notfiy_master/grpc_notify.h"
+#include "worker/notfiy_master/local_notify.h"
+
+namespace mindspore::serving {
+
+void PyWorker::StartServable(const std::string &model_directory, const std::string &model_name, uint32_t version_number,
+                             const std::string &master_ip, uint32_t master_port, const std::string &host_ip,
+                             uint32_t host_port) {
+  auto notify_master = std::make_shared<GrpcNotfiyMaster>(master_ip, master_port, host_ip, host_port);
+  auto status = Worker::GetInstance().StartServable(model_directory, model_name, version_number, notify_master);
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+  status = Worker::GetInstance().StartGrpcServer(host_ip, host_port);
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+  status = Worker::GetInstance().StartVersionController();
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+}
+
+void PyWorker::StartServableInMaster(const std::string &model_directory, const std::string &model_name,
+                                     uint32_t version_number) {
+  auto notify_master = std::make_shared<LocalNotifyMaster>();
+  auto status = Worker::GetInstance().StartServable(model_directory, model_name, version_number, notify_master);
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+  status = Worker::GetInstance().StartVersionController();
+  if (status != SUCCESS) {
+    MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
+  }
+}
+
+TaskItem PyWorker::GetPyTask() {
+  TaskItem item;
+  Worker::GetInstance().GetPyTaskQueueGroup().PopPyTask(item);
+  return item;
+}
+
+TaskItem PyWorker::TryGetPreprocessPyTask() {
+  TaskItem item;
+  Worker::GetInstance().GetPyTaskQueueGroup().TryPopPreprocessTask(item);
+  return item;
+}
+
+TaskItem PyWorker::TryGetPostprocessPyTask() {
+  TaskItem item;
+  Worker::GetInstance().GetPyTaskQueueGroup().TryPopPostprocessTask(item);
+  return item;
+}
+
+void PyWorker::PushPreprocessPyResult(const py::tuple &output_batch) {
+  MSI_TIME_STAMP_START(PushPreprocessPyResult)
+  std::vector<ResultInstance> outputs;
+  for (auto &output : output_batch) {
+    ResultInstance instance;
+    instance.data = PyTensor::AsInstanceData(py::cast<py::tuple>(output));
+    outputs.push_back(instance);
+  }
+  Worker::GetInstance().GetPyTaskQueuePreprocess()->PushTaskPyResult(outputs);
+  MSI_TIME_STAMP_END(PushPreprocessPyResult)
+}
+
+void PyWorker::PushPreprocessPyFailed(int count) {
+  std::vector<ResultInstance> results;
+  Status error_msg(INVALID_INPUTS, "Preprocess Failed");
+  for (int i = 0; i < count; i++) {
+    ResultInstance result_instance;
+    result_instance.error_msg = error_msg;
+    results.push_back(result_instance);
+  }
+  Worker::GetInstance().GetPyTaskQueuePreprocess()->PushTaskPyResult(results);
+}
+
+void PyWorker::PushPostprocessPyResult(const py::tuple &output_batch) {
+  std::vector<ResultInstance> outputs;
+  for (auto &output : output_batch) {
+    ResultInstance instance;
+    instance.data = PyTensor::AsInstanceData(py::cast<py::tuple>(output));
+    outputs.push_back(instance);
+  }
+  Worker::GetInstance().GetPyTaskQueuePostprocess()->PushTaskPyResult(outputs);
+}
+
+void PyWorker::PushPostprocessPyFailed(int count) {
+  std::vector<ResultInstance> results;
+  Status error_msg(INVALID_INPUTS, "Postprocess Failed");
+  for (int i = 0; i < count; i++) {
+    ResultInstance result_instance;
+    result_instance.error_msg = error_msg;
+    results.push_back(result_instance);
+  }
+  Worker::GetInstance().GetPyTaskQueuePostprocess()->PushTaskPyResult(results);
+}
+
+void PyWorker::WaitAndClear() {
+  ExitHandle::Instance().WorkerWait();
+  Worker::GetInstance().Clear();
+}
+
+void PyWorker::Stop() { ExitHandle::Instance().Stop(); }
+
+int PyWorker::GetBatchSize() { return Worker::GetInstance().GetBatchSize(); }
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/python/worker/worker_py.h

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_WORKER_PY_H
+#define MINDSPORE_SERVING_WORKER_PY_H
+
+#include <string>
+#include <unordered_map>
+#include "worker/worker.h"
+#include "worker/task_queue.h"
+#include "python/tensor_py.h"
+
+namespace mindspore::serving {
+
+class MS_API PyWorker {
+ public:
+  static void StartServable(const std::string &model_directory, const std::string &model_name, uint32_t version_number,
+                            const std::string &master_ip, uint32_t master_port, const std::string &host_ip,
+                            uint32_t host_port);
+
+  static void StartServableInMaster(const std::string &model_directory, const std::string &model_name,
+                                    uint32_t version_number);
+
+  static int GetBatchSize();
+  static void WaitAndClear();
+  static void Stop();
+  static TaskItem GetPyTask();
+  static TaskItem TryGetPreprocessPyTask();
+  static TaskItem TryGetPostprocessPyTask();
+  static void PushPreprocessPyResult(const py::tuple &output_batch);
+  static void PushPreprocessPyFailed(int count);
+
+  static void PushPostprocessPyResult(const py::tuple &output_batch);
+  static void PushPostprocessPyFailed(int count);
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_WORKER_PY_H

mindspore_serving/ccsrc/worker/context.cc

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "worker/context.h"
+
+namespace mindspore::serving {
+
+std::shared_ptr<ServableContext> ServableContext::Instance() {
+  static std::shared_ptr<ServableContext> instance;
+  if (instance == nullptr) {
+    instance = std::make_shared<ServableContext>();
+  }
+  return instance;
+}
+
+void ServableContext::SetDeviceType(DeviceType device_type) { device_type_ = device_type; }
+
+DeviceType ServableContext::GetDeviceType() const { return device_type_; }
+
+void ServableContext::SetDeviceId(uint32_t device_id) { device_id_ = device_id; }
+
+uint32_t ServableContext::GetDeviceId() const { return device_id_; }
+
+Status ServableContext::SetDeviceTypeStr(const std::string &device_type) {
+  DeviceType type;
+  std::string device_type_lowcase = device_type;
+  for (auto &c : device_type_lowcase) {
+    if (c >= 'A' && c <= 'Z') {
+      c = c - 'A' + 'a';
+    }
+  }
+  if (device_type_lowcase == "ascend" || device_type_lowcase == "davinci") {
+    type = kDeviceTypeAscend;
+  } else if (device_type_lowcase == "gpu") {
+    type = kDeviceTypeGpu;
+  } else if (device_type_lowcase == "cpu") {
+    type = kDeviceTypeCpu;
+  } else if (device_type_lowcase == "none") {
+    type = kDeviceTypeNotSpecified;
+  } else {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Unsupport device type " << device_type;
+  }
+  SetDeviceType(type);
+  return SUCCESS;
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/worker/context.h

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_CONTEXT_H
+#define MINDSPORE_CONTEXT_H
+
+#include <string>
+#include <memory>
+#include <vector>
+#include "common/serving_common.h"
+#include "worker/inference/inference.h"
+
+namespace mindspore::serving {
+
+class MS_API ServableContext {
+ public:
+  static std::shared_ptr<ServableContext> Instance();
+
+  Status SetDeviceTypeStr(const std::string &device_type);
+
+  void SetDeviceType(DeviceType device_type);
+  DeviceType GetDeviceType() const;
+  void SetDeviceId(uint32_t device_id);
+  uint32_t GetDeviceId() const;
+
+ private:
+  DeviceType device_type_ = kDeviceTypeNotSpecified;
+  uint32_t device_id_ = 0;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_CONTEXT_H

mindspore_serving/ccsrc/worker/grpc/worker_process.cc

@@ -0,0 +1,68 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "worker/grpc/worker_process.h"
+#include "master/dispacther.h"
+#include "worker/worker.h"
+
+namespace mindspore {
+namespace serving {
+grpc::Status MSWorkerImpl::Exit(grpc::ServerContext *context, const proto::ExitRequest *request,
+                                proto::ExitReply *reply) {
+  MSI_LOG(INFO) << "Master Exit";
+  Worker::GetInstance().StopServable(false);
+  return grpc::Status::OK;
+}
+
+grpc::Status MSWorkerImpl::Predict(grpc::ServerContext *context, const proto::PredictRequest *request,
+                                   proto::PredictReply *reply) {
+  Status status(FAILED);
+  MSI_LOG(INFO) << "Begin call service Eval";
+  try {
+    MSI_TIME_STAMP_START(Predict)
+    status = Worker::GetInstance().Run(*request, *reply);
+    MSI_TIME_STAMP_END(Predict)
+  } catch (const std::bad_alloc &ex) {
+    MSI_LOG(ERROR) << "Serving Error: malloc memory failed";
+    std::cout << "Serving Error: malloc memory failed" << std::endl;
+  } catch (const std::runtime_error &ex) {
+    MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what();
+    std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl;
+  } catch (const std::exception &ex) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what();
+    std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl;
+  } catch (...) {
+    MSI_LOG(ERROR) << "Serving Error: exception occurred";
+    std::cout << "Serving Error: exception occurred";
+  }
+  MSI_LOG(INFO) << "Finish call service Eval";
+
+  if (status == INVALID_INPUTS) {
+    auto proto_error_msg = reply->add_error_msg();
+    proto_error_msg->set_error_code(status.StatusCode());
+    proto_error_msg->set_error_msg(status.StatusMessage());
+    return grpc::Status::OK;
+  } else if (status != SUCCESS) {
+    auto proto_error_msg = reply->add_error_msg();
+    proto_error_msg->set_error_code(FAILED);
+    proto_error_msg->set_error_msg("Predict failed");
+    return grpc::Status::OK;
+  }
+  return grpc::Status::OK;
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/worker/grpc/worker_process.h

@@ -0,0 +1,41 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_WORKER_PROCESS_H
+#define MINDSPORE_WORKER_PROCESS_H
+
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/health_check_service_interface.h>
+#include <grpcpp/ext/proto_server_reflection_plugin.h>
+#include "common/serving_common.h"
+#include "proto/ms_worker.pb.h"
+#include "proto/ms_worker.grpc.pb.h"
+
+namespace mindspore {
+namespace serving {
+
+// Service Implement
+class MSWorkerImpl final : public proto::MSWorker::Service {
+ public:
+  grpc::Status Predict(grpc::ServerContext *context, const proto::PredictRequest *request,
+                       proto::PredictReply *reply) override;
+  grpc::Status Exit(grpc::ServerContext *context, const proto::ExitRequest *request, proto::ExitReply *reply) override;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_WORKER_PROCESS_H

mindspore_serving/ccsrc/worker/inference/inference.h

@@ -0,0 +1,209 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_INC_INFERENCE_H
+#define MINDSPORE_SERVING_INC_INFERENCE_H
+
+#include <utility>
+#include <unordered_map>
+#include <memory>
+#include <vector>
+#include <string>
+#include "common/tensor_base.h"
+#include "common/tensor.h"
+#include "common/log.h"
+#include "common/status.h"
+#include "include/api/types.h"
+
+namespace mindspore {
+namespace serving {
+
+using api::ModelType;
+using api::ModelType::kMindIR;
+using api::ModelType::kOM;
+
+struct TensorInfo {
+  size_t size;  // -1: unspecified
+  DataType data_type;
+  std::vector<int64_t> shape;
+};
+
+enum DeviceType {
+  kDeviceTypeNotSpecified,
+  kDeviceTypeAscendMS,
+  kDeviceTypeAscendCL,
+  kDeviceTypeAscend,
+  kDeviceTypeGpu,
+  kDeviceTypeCpu,
+};
+
+class MS_API InferSession {
+ public:
+  InferSession() = default;
+  virtual ~InferSession() = default;
+  virtual Status InitEnv(DeviceType device_type, uint32_t device_id,
+                         const std::unordered_map<std::string, std::string> &other_options) = 0;
+  virtual Status FinalizeEnv() = 0;
+
+  virtual Status LoadModelFromFile(serving::DeviceType device_type, uint32_t device_id, const std::string &file_name,
+                                   ModelType model_type, uint32_t &model_id) = 0;
+  virtual Status UnloadModel(uint32_t model_id) = 0;
+  // override this method to avoid request/reply data copy
+  virtual Status ExecuteModel(uint32_t model_id, const RequestBase &request, ReplyBase &reply) = 0;
+  virtual Status ExecuteModel(uint32_t model_id, const std::vector<TensorBasePtr> &request,
+                              std::vector<TensorBasePtr> &reply) {
+    VectorTensorPtrWrapRequest wrap_request(request);
+    VectorTensorPtrWrapReply wrap_reply(reply, []() { return std::make_shared<Tensor>(); });
+    return ExecuteModel(model_id, wrap_request, wrap_reply);
+  }
+
+  virtual std::vector<TensorInfo> GetInputInfos(uint32_t model_id) const = 0;
+  virtual std::vector<TensorInfo> GetOutputInfos(uint32_t model_id) const = 0;
+  virtual ssize_t GetBatchSize(uint32_t model_id) const = 0;
+  virtual TensorBasePtr MakeInferenceTensor(DataType data_type, const std::vector<int64_t> &shape) const {
+    return nullptr;
+  }
+  virtual bool CheckModelSupport(DeviceType device_type, ModelType model_type) const { return true; }
+};
+
+struct InferSessionRegInfo {
+  std::shared_ptr<InferSession> session;
+  ModelType model_type;
+  int priority;
+};
+
+class MS_API InferSessionStorage {
+ public:
+  void Register(DeviceType device_type, ModelType model_type, const std::shared_ptr<InferSession> &session,
+                int priority) {
+    auto &list = session_map_[device_type];
+    InferSessionRegInfo info{session, model_type, priority};
+    list.push_back(info);
+  }
+
+  std::shared_ptr<InferSession> Get(DeviceType device_type, ModelType model_type, DeviceType *specified_device_type) {
+    MSI_EXCEPTION_IF_NULL(specified_device_type);
+    if (device_type == kDeviceTypeNotSpecified) {
+      for (auto &item_device : session_map_) {
+        std::shared_ptr<InferSession> ret_session = GetSession(item_device.second, item_device.first, model_type);
+        if (ret_session) {
+          *specified_device_type = item_device.first;
+          return ret_session;
+        }
+      }
+      return nullptr;
+    } else if (device_type == kDeviceTypeAscend) {
+      auto ascend_list = {kDeviceTypeAscendCL, kDeviceTypeAscendMS};
+      for (auto ascend_type : ascend_list) {
+        auto it = session_map_.find(ascend_type);
+        if (it == session_map_.end()) {
+          continue;
+        }
+        auto session_ret = GetSession(it->second, ascend_type, model_type);
+        if (session_ret != nullptr) {
+          *specified_device_type = ascend_type;
+          return session_ret;
+        }
+      }
+      return nullptr;
+    }
+    auto it = session_map_.find(device_type);
+    if (it == session_map_.end()) {
+      return nullptr;
+    }
+    std::shared_ptr<InferSession> session_ret;
+    session_ret = GetSession(it->second, device_type, model_type);
+    *specified_device_type = device_type;
+    return session_ret;
+  }
+
+  static InferSessionStorage &Instance() {
+    static InferSessionStorage instance;
+    return instance;
+  }
+
+ private:
+  std::unordered_map<DeviceType, std::vector<InferSessionRegInfo>> session_map_;
+
+  std::shared_ptr<InferSession> GetSession(const std::vector<InferSessionRegInfo> &session_list, DeviceType device_type,
+                                           ModelType model_type) {
+    std::shared_ptr<InferSession> session_ret = nullptr;
+    int cur_priority = INT32_MIN;
+    for (auto &item : session_list) {
+      if (item.model_type != model_type) {
+        continue;
+      }
+      if (session_ret == nullptr || cur_priority < item.priority) {
+        if (!item.session->CheckModelSupport(device_type, model_type)) {
+          MSI_LOG_INFO << "CheckModelSupport for " << device_type << " " << model_type << " failed, skipped";
+          continue;
+        }
+        cur_priority = item.priority;
+        session_ret = item.session;
+      }
+    }
+    return session_ret;
+  }
+};
+
+class MS_API InferSessionRegister {
+ public:
+  InferSessionRegister(DeviceType device_type, ModelType model_type, const std::shared_ptr<InferSession> &session,
+                       int priority) {
+    InferSessionStorage::Instance().Register(device_type, model_type, session, priority);
+  }
+};
+
+#define REGISTER_INFER_SEESION_UNIQUE(device_type, model_type, cls_name, priority, index)  \
+  static mindspore::serving::InferSessionRegister g_register_session_##cls_name##_##index( \
+    device_type, model_type, std::make_shared<cls_name>(), priority);
+
+#define REGISTER_INFER_SEESION_HELPER(device_type, model_type, cls_name, priority, index) \
+  REGISTER_INFER_SEESION_UNIQUE(device_type, model_type, cls_name, priority, index)
+
+#define REGISTER_INFER_SEESION(device_type, model_type, cls_name, priority) \
+  REGISTER_INFER_SEESION_HELPER(device_type, model_type, cls_name, priority, __COUNTER__);
+
+static inline LogStream &operator<<(LogStream &stream, DeviceType device_type) {
+  switch (device_type) {
+    case kDeviceTypeAscend:
+      stream << "kDeviceTypeAscend";
+      break;
+    case kDeviceTypeAscendMS:
+      stream << "kDeviceTypeAscend910";
+      break;
+    case kDeviceTypeAscendCL:
+      stream << "kDeviceTypeAscend310";
+      break;
+    case kDeviceTypeGpu:
+      stream << "kDeviceTypeGpu";
+      break;
+    case kDeviceTypeCpu:
+      stream << "kDeviceTypeCpu";
+      break;
+    case kDeviceTypeNotSpecified:
+      stream << "kDeviceTypeNotSpecified";
+      break;
+    default:
+      stream << "[device type " << static_cast<int>(device_type) << "]";
+      break;
+  }
+  return stream;
+}
+
+}  // namespace serving
+}  // namespace mindspore
+#endif  // MINDSPORE_SERVING_INC_INFERENCE_H

mindspore_serving/ccsrc/worker/inference/mindspore_model_wrap.cc

@@ -0,0 +1,342 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mindspore_model_wrap.h"
+#include "api/types.h"
+
+namespace mindspore {
+namespace serving {
+
+Status MindSporeModelWrap::InitEnv(serving::DeviceType device_type, uint32_t device_id,
+                                   const std::unordered_map<std::string, std::string> &other_options) {
+  return SUCCESS;
+}
+
+Status MindSporeModelWrap::FinalizeEnv() { return SUCCESS; }
+
+api::DataType TransInferDataType2ApiTypeId(DataType data_type) {
+  const std::map<DataType, api::DataType> type2id_map{
+    {serving::kMSI_Unknown, api::kMsUnknown}, {serving::kMSI_Bool, api::kMsBool},
+    {serving::kMSI_Int8, api::kMsInt8},       {serving::kMSI_Uint8, api::kMsUint8},
+    {serving::kMSI_Int16, api::kMsInt16},     {serving::kMSI_Uint16, api::kMsUint16},
+    {serving::kMSI_Int32, api::kMsInt32},     {serving::kMSI_Uint32, api::kMsUint32},
+    {serving::kMSI_Int64, api::kMsInt64},     {serving::kMSI_Uint64, api::kMsUint64},
+    {serving::kMSI_Float16, api::kMsFloat16}, {serving::kMSI_Float32, api::kMsFloat32},
+    {serving::kMSI_Float64, api::kMsFloat64},
+  };
+  auto it = type2id_map.find(data_type);
+  if (it == type2id_map.end()) {
+    MSI_LOG_WARNING << "Unsupported MSI data type " << data_type;
+    return api::kMsUnknown;
+  } else {
+    return it->second;
+  }
+}
+
+DataType TransTypeId2InferDataType(api::DataType type_id) {
+  const std::map<api::DataType, DataType> id2type_map{
+    {api::kMsUnknown, kMSI_Unknown}, {api::kMsBool, kMSI_Bool},     {api::kMsFloat64, kMSI_Float64},
+    {api::kMsInt8, kMSI_Int8},       {api::kMsUint8, kMSI_Uint8},   {api::kMsInt16, kMSI_Int16},
+    {api::kMsUint16, kMSI_Uint16},   {api::kMsInt32, kMSI_Int32},   {api::kMsUint32, kMSI_Uint32},
+    {api::kMsInt64, kMSI_Int64},     {api::kMsUint64, kMSI_Uint64}, {api::kMsFloat16, kMSI_Float16},
+    {api::kMsFloat32, kMSI_Float32},
+  };
+  auto it = id2type_map.find(type_id);
+  if (it == id2type_map.end()) {
+    MSI_LOG_WARNING << "Unsupported data id " << type_id;
+    return kMSI_Unknown;
+  } else {
+    return it->second;
+  }
+}
+
+Status MindSporeModelWrap::LoadModelFromFile(serving::DeviceType device_type, uint32_t device_id,
+                                             const std::string &file_name, ModelType model_type, uint32_t &model_id) {
+  std::string device_type_str;
+  if (device_type == kDeviceTypeAscendMS) {
+    device_type_str = api::kDeviceTypeAscendMS;
+  } else if (device_type == kDeviceTypeAscendCL) {
+    device_type_str = api::kDeviceTypeAscendCL;
+  } else {
+    MSI_LOG_EXCEPTION << "Only support Ascend310 or Ascend910 in MindSporeModelWrap";
+  }
+
+  std::shared_ptr<api::Model> model = nullptr;
+  try {
+    model = std::make_shared<api::Model>(device_type_str, device_id);
+  } catch (std::runtime_error &ex) {
+    MSI_LOG_ERROR << "Load model from file failed, device_type " << device_type_str << ", device_id " << device_id;
+    return FAILED;
+  }
+  api::ModelType api_model_type;
+  switch (model_type) {
+    case kMindIR:
+      api_model_type = api::kMindIR;
+      break;
+    case kOM:
+      api_model_type = api::kOM;
+      break;
+    default:
+      MSI_LOG_EXCEPTION << "Only support OM and MindIR, now model type is " << model_type;
+  }
+  api::Status status = model->LoadModel(file_name, api_model_type, {});
+  if (!status.IsSuccess()) {
+    return Status(FAILED, status.StatusMessage());
+  }
+  model_index_++;
+  model_id = model_index_;
+  ApiModelInfo api_model_info;
+  api_model_info.model = model;
+  api_model_info.device_type = device_type_str;
+  api_model_info.device_id = device_id;
+  auto st = GetModelInfos(api_model_info);
+  if (st != SUCCESS) {
+    return st;
+  }
+  model_map_[model_id] = api_model_info;
+  return SUCCESS;
+}
+
+Status MindSporeModelWrap::GetModelInfos(ApiModelInfo &api_model_info) {
+  std::vector<api::Tensor> input_tensors;
+  auto model = api_model_info.model;
+  api::Status status = model->GetInputsInfo(&input_tensors);
+  if (!status.IsSuccess()) {
+    return Status(FAILED, status.StatusMessage());
+  }
+  bool first_dim_same = true;
+  auto find_batch_size = [&first_dim_same, &api_model_info](const std::vector<int64_t> &shape) {
+    if (first_dim_same) {
+      if (shape.empty()) {
+        first_dim_same = false;
+      } else if (api_model_info.batch_size != 0) {
+        if (api_model_info.batch_size != shape[0]) {
+          first_dim_same = false;
+        }
+      } else {
+        api_model_info.batch_size = shape[0];
+      }
+    }
+  };
+  auto shape_element_num = [](const std::vector<int64_t> &shape) -> size_t {
+    size_t elements_nums = std::accumulate(shape.begin(), shape.end(), 1LL, std::multiplies<size_t>());
+    return elements_nums;
+  };
+  auto get_tensor_info_from_tensor = [find_batch_size, shape_element_num](const api::Tensor &tensor) {
+    serving::TensorInfo tensor_info;
+    tensor_info.shape = tensor.Shape();
+    tensor_info.data_type = TransTypeId2InferDataType(tensor.DataType());
+    tensor_info.size = tensor.DataSize();
+    if (tensor_info.size == 0) {
+      tensor_info.size = TensorBase::GetTypeSize(tensor_info.data_type) * shape_element_num(tensor_info.shape);
+    }
+    find_batch_size(tensor_info.shape);
+    return tensor_info;
+  };
+  for (auto &item : input_tensors) {
+    api_model_info.input_names.push_back(item.Name());
+    auto tensor_info = get_tensor_info_from_tensor(item);
+    if (tensor_info.data_type == kMSI_Unknown) {
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Unknown input api data type " << item.DataType();
+    }
+    api_model_info.input_tensor_infos.push_back(tensor_info);
+  }
+  std::vector<api::Tensor> output_tensors;
+  status = model->GetOutputsInfo(&output_tensors);
+  if (!status.IsSuccess()) {
+    return Status(FAILED, status.StatusMessage());
+  }
+  for (auto &item : output_tensors) {
+    api_model_info.output_names.push_back(item.Name());
+    auto tensor_info = get_tensor_info_from_tensor(item);
+    if (tensor_info.data_type == kMSI_Unknown) {
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Unknown output api data type " << item.DataType();
+    }
+    api_model_info.output_tensor_infos.push_back(tensor_info);
+  }
+  if (!first_dim_same) {
+    api_model_info.batch_size = 0;
+  }
+  return SUCCESS;
+}
+
+Status MindSporeModelWrap::UnloadModel(uint32_t model_id) {
+  auto it = model_map_.find(model_id);
+  if (it == model_map_.end()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Invalid model id " << model_id;
+  }
+  auto model = it->second.model;
+  api::Status status = model->UnloadModel();
+  model_map_.erase(it);
+  if (!status.IsSuccess()) {
+    return Status(FAILED, status.StatusMessage());
+  }
+  return SUCCESS;
+}
+
+Status MindSporeModelWrap::ExecuteModel(uint32_t model_id, const RequestBase &request, serving::ReplyBase &reply) {
+  FuncMakeInBuffer func_in = [&request](size_t index) {
+    auto input_tensor = request[index];
+    return api::Buffer(input_tensor->data(), input_tensor->data_size());
+  };
+
+  FuncMakeOutTensor func_out = [&reply](const api::Buffer &result_tensor, DataType data_type,
+                                        const std::vector<int64_t> &shape) {
+    auto tensor = reply.add();
+    MSI_EXCEPTION_IF_NULL(tensor);
+    tensor->set_data(result_tensor.Data(), result_tensor.DataSize());
+    tensor->set_data_type(data_type);
+    tensor->set_shape(shape);
+  };
+  return ExecuteModelCommon(model_id, request.size(), func_in, func_out);
+}
+
+Status MindSporeModelWrap::ExecuteModel(uint32_t model_id, const std::vector<TensorBasePtr> &request,
+                                        std::vector<TensorBasePtr> &reply) {
+  FuncMakeInBuffer func_in = [&request](size_t index) {
+    auto &input_tensor = request[index];
+    auto api_buffer_wrap = std::dynamic_pointer_cast<ApiBufferTensorWrap>(input_tensor);
+    if (api_buffer_wrap) {
+      return api_buffer_wrap->GetBuffer();
+    } else {
+      return api::Buffer(input_tensor->data(), input_tensor->data_size());
+    }
+  };
+  FuncMakeOutTensor func_out = [&reply](const api::Buffer &result_tensor, DataType data_type,
+                                        const std::vector<int64_t> &shape) {
+    auto tensor = std::make_shared<ApiBufferTensorWrap>(result_tensor);
+    tensor->set_data_type(data_type);
+    tensor->set_shape(shape);
+    reply.push_back(tensor);
+  };
+  return ExecuteModelCommon(model_id, request.size(), func_in, func_out);
+}
+
+Status MindSporeModelWrap::ExecuteModelCommon(uint32_t model_id, size_t request_size, const FuncMakeInBuffer &in_func,
+                                              const FuncMakeOutTensor &out_func) {
+  auto it = model_map_.find(model_id);
+  if (it == model_map_.end()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Invalid model id " << model_id;
+  }
+  auto &model_info = it->second;
+  auto model = model_info.model;
+  auto &input_names = model_info.input_names;
+  auto &output_names = model_info.output_names;
+  if (input_names.size() != request_size) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Inputs size not match, request inputs size " << request_size
+                                          << ", model inputs size " << input_names.size();
+  }
+  std::map<std::string, api::Buffer> inputs;
+  for (size_t i = 0; i < input_names.size(); i++) {
+    inputs[input_names[i]] = in_func(i);
+  }
+  std::map<std::string, api::Buffer> outputs;
+  api::Status status = model->Predict(inputs, &outputs);
+  if (!status.IsSuccess()) {
+    MSI_LOG_ERROR << "Predict failed: " << status.StatusMessage();
+    return FAILED;
+  }
+  if (outputs.size() != output_names.size()) {
+    return INFER_STATUS_LOG_ERROR(FAILED) << "Outputs size not match, predict outputs size " << outputs.size()
+                                          << ", model outputs size " << output_names.size();
+  }
+  auto &output_infos = model_info.output_tensor_infos;
+  for (size_t i = 0; i < output_names.size(); i++) {
+    auto &output_name = output_names[i];
+    auto output_it = outputs.find(output_name);
+    if (output_it == outputs.end()) {
+      return INFER_STATUS_LOG_ERROR(FAILED)
+             << "Get output failed, cannot find output " << output_name << " in predict result";
+    }
+    auto &result_tensor = output_it->second;
+    auto &output_info = output_infos[i];
+    if (result_tensor.DataSize() != output_info.size) {
+      return INFER_STATUS_LOG_ERROR(FAILED)
+             << "Get output failed, predict output data size " << result_tensor.DataSize()
+             << " not match model info data size " << output_info.size << ", output_name " << output_name;
+    }
+    out_func(result_tensor, output_info.data_type, output_info.shape);
+  }
+  return SUCCESS;
+}
+
+std::vector<serving::TensorInfo> MindSporeModelWrap::GetInputInfos(uint32_t model_id) const {
+  auto it = model_map_.find(model_id);
+  if (it == model_map_.end()) {
+    MSI_LOG_ERROR << "Invalid model id " << model_id;
+    return {};
+  }
+  auto &model_info = it->second;
+  return model_info.input_tensor_infos;
+}
+
+std::vector<serving::TensorInfo> MindSporeModelWrap::GetOutputInfos(uint32_t model_id) const {
+  auto it = model_map_.find(model_id);
+  if (it == model_map_.end()) {
+    MSI_LOG_ERROR << "Invalid model id " << model_id;
+    return {};
+  }
+  auto &model_info = it->second;
+  return model_info.output_tensor_infos;
+}
+
+ssize_t MindSporeModelWrap::GetBatchSize(uint32_t model_id) const {
+  auto it = model_map_.find(model_id);
+  if (it == model_map_.end()) {
+    MSI_LOG_ERROR << "Invalid model id " << model_id;
+    return {};
+  }
+  auto &model_info = it->second;
+  return model_info.batch_size;
+}
+
+TensorBasePtr MindSporeModelWrap::MakeInferenceTensor(DataType data_type, const std::vector<int64_t> &shape) const {
+  return std::make_shared<ApiBufferTensorWrap>(data_type, shape);
+}
+
+bool MindSporeModelWrap::CheckModelSupport(DeviceType device_type, ModelType model_type) const {
+  std::string device_type_str;
+  switch (device_type) {
+    case kDeviceTypeAscendMS:
+      device_type_str = api::kDeviceTypeAscendMS;
+      break;
+    case kDeviceTypeAscendCL:
+      device_type_str = api::kDeviceTypeAscendCL;
+      break;
+    default:
+      return false;
+  }
+  return api::Model::CheckModelSupport(device_type_str, model_type);
+}
+
+ApiBufferTensorWrap::ApiBufferTensorWrap() = default;
+
+ApiBufferTensorWrap::ApiBufferTensorWrap(DataType type, const std::vector<int64_t> &shape)
+    : type_(type), shape_(shape) {
+  size_t data_len = itemsize() * TensorBase::element_cnt();
+  buffer_.ResizeData(data_len);
+}
+
+ApiBufferTensorWrap::ApiBufferTensorWrap(const api::Buffer &buffer) : buffer_(buffer) {}
+
+ApiBufferTensorWrap::~ApiBufferTensorWrap() = default;
+
+REGISTER_INFER_SEESION(serving::kDeviceTypeAscendCL, kOM, MindSporeModelWrap, 1);
+REGISTER_INFER_SEESION(serving::kDeviceTypeAscendCL, kMindIR, MindSporeModelWrap, 1);
+REGISTER_INFER_SEESION(serving::kDeviceTypeAscendMS, kMindIR, MindSporeModelWrap, 1);
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/worker/inference/mindspore_model_wrap.h

@@ -0,0 +1,122 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_WROERK_ACL_INFERENCE_H
+#define MINDSPORE_SERVING_WROERK_ACL_INFERENCE_H
+
+#include <unordered_map>
+#include "common/serving_common.h"
+#include "worker/inference/inference.h"
+#include "api/model.h"
+
+namespace mindspore {
+namespace serving {
+
+struct ApiModelInfo {
+  std::vector<std::string> input_names;
+  std::vector<serving::TensorInfo> input_tensor_infos;
+  std::vector<std::string> output_names;
+  std::vector<serving::TensorInfo> output_tensor_infos;
+  std::shared_ptr<api::Model> model;
+  uint32_t batch_size = 0;
+  std::string device_type;
+  uint32_t device_id = 0;
+};
+
+class MindSporeModelWrap : public InferSession {
+ public:
+  MindSporeModelWrap() = default;
+
+  ~MindSporeModelWrap() = default;
+
+  Status InitEnv(serving::DeviceType device_type, uint32_t device_id,
+                 const std::unordered_map<std::string, std::string> &other_options) override;
+
+  Status FinalizeEnv() override;
+
+  Status LoadModelFromFile(serving::DeviceType device_type, uint32_t device_id, const std::string &file_name,
+                           serving::ModelType model_type, uint32_t &model_id) override;
+
+  Status UnloadModel(uint32_t model_id) override;
+
+  // override this method to avoid request/reply data copy
+  Status ExecuteModel(uint32_t model_id, const RequestBase &request, ReplyBase &reply) override;
+  Status ExecuteModel(uint32_t model_id, const std::vector<TensorBasePtr> &request,
+                      std::vector<TensorBasePtr> &reply) override;
+
+  std::vector<serving::TensorInfo> GetInputInfos(uint32_t model_id) const override;
+
+  std::vector<serving::TensorInfo> GetOutputInfos(uint32_t model_id) const override;
+
+  ssize_t GetBatchSize(uint32_t model_id) const override;
+
+  TensorBasePtr MakeInferenceTensor(DataType data_type, const std::vector<int64_t> &shape) const override;
+
+  bool CheckModelSupport(DeviceType device_type, ModelType model_type) const override;
+
+ private:
+  std::unordered_map<uint32_t, ApiModelInfo> model_map_;
+  uint32_t model_index_ = 0;
+
+  using FuncMakeInBuffer = std::function<api::Buffer(size_t index)>;
+  using FuncMakeOutTensor =
+    std::function<void(const api::Buffer, DataType data_type, const std::vector<int64_t> &shape)>;
+  Status ExecuteModelCommon(uint32_t model_id, size_t request_size, const FuncMakeInBuffer &in_func,
+                            const FuncMakeOutTensor &out_func);
+  Status GetModelInfos(ApiModelInfo &model_info);
+};
+
+class ApiBufferTensorWrap : public TensorBase {
+ public:
+  ApiBufferTensorWrap();
+  ApiBufferTensorWrap(DataType type, const std::vector<int64_t> &shape);
+  explicit ApiBufferTensorWrap(const api::Buffer &buffer);
+  ~ApiBufferTensorWrap() override;
+
+  void set_data_type(DataType type) override { type_ = type; }
+  DataType data_type() const override { return type_; }
+
+  void set_shape(const std::vector<int64_t> &shape) override { shape_ = shape; }
+  std::vector<int64_t> shape() const override { return shape_; }
+
+  const uint8_t *data() const override { return static_cast<const uint8_t *>(buffer_.Data()); }
+  size_t data_size() const override { return buffer_.DataSize(); }
+
+  bool resize_data(size_t data_len) override { return buffer_.ResizeData(data_len); }
+  uint8_t *mutable_data() override { return static_cast<uint8_t *>(buffer_.MutableData()); }
+
+  // For kMSI_String and kMSI_Bytes
+  void clear_bytes_data() override { MSI_LOG_EXCEPTION << "Not support for api::Buffer Tensor"; }
+  void add_bytes_data(const uint8_t *data, size_t bytes_len) override {
+    MSI_LOG_EXCEPTION << "Not support for api::Buffer Tensor";
+  }
+  size_t bytes_data_size() const override { MSI_LOG_EXCEPTION << "Not support for api::Buffer Tensor"; }
+  void get_bytes_data(size_t index, const uint8_t *&data, size_t &bytes_len) const override {
+    MSI_LOG_EXCEPTION << "Not support for api::Buffer Tensor";
+  }
+
+  api::Buffer GetBuffer() const { return buffer_; }
+
+ private:
+  DataType type_ = kMSI_Unknown;
+  std::vector<int64_t> shape_;
+  api::Buffer buffer_;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVING_WROERK_ACL_INFERENCE_H

mindspore_serving/ccsrc/worker/model.cc

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "worker/model.h"
+#include <algorithm>
+#include "mindspore_serving/ccsrc/common/tensor.h"
+
+namespace mindspore::serving {
+
+Status AscendModelServable::Predict(const std::vector<TensorBasePtr> &input, std::vector<TensorBasePtr> &output) {
+  return session_->ExecuteModel(model_id_, input, output);
+}
+
+std::vector<TensorInfo> AscendModelServable::GetInputInfos() const { return session_->GetInputInfos(model_id_); }
+
+std::vector<TensorInfo> AscendModelServable::GetOutputInfos() const { return session_->GetOutputInfos(model_id_); }
+
+uint64_t AscendModelServable::GetBatchSize() const { return session_->GetBatchSize(model_id_); }
+
+TensorBasePtr AscendModelServable::MakeInferenceTensor(DataType data_type, const std::vector<int64_t> &shape) const {
+  return session_->MakeInferenceTensor(data_type, shape);
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/worker/model.h

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_MODEL_H
+#define MINDSPORE_SERVING_MODEL_H
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include <string>
+
+#include "common/serving_common.h"
+#include "common/instance.h"
+#include "mindspore_serving/ccsrc/worker/inference/inference.h"
+#include "common/servable.h"
+
+namespace mindspore::serving {
+
+class ServableBase {
+ public:
+  ServableBase() = default;
+  virtual ~ServableBase() = default;
+
+  virtual Status Predict(const std::vector<TensorBasePtr> &input, std::vector<TensorBasePtr> &output) = 0;
+
+  virtual std::vector<TensorInfo> GetInputInfos() const = 0;
+  virtual std::vector<TensorInfo> GetOutputInfos() const = 0;
+  virtual uint64_t GetBatchSize() const = 0;
+  virtual TensorBasePtr MakeInferenceTensor(DataType data_type, const std::vector<int64_t> &shape) const {
+    return nullptr;
+  }
+};
+
+class AscendModelServable : public ServableBase {
+ public:
+  AscendModelServable(const std::shared_ptr<serving::InferSession> &session, uint32_t model_id)
+      : session_(session), model_id_(model_id) {}
+  ~AscendModelServable() = default;
+
+  Status Predict(const std::vector<TensorBasePtr> &input, std::vector<TensorBasePtr> &output) override;
+
+  std::vector<TensorInfo> GetInputInfos() const override;
+  std::vector<TensorInfo> GetOutputInfos() const override;
+  uint64_t GetBatchSize() const override;
+  TensorBasePtr MakeInferenceTensor(DataType data_type, const std::vector<int64_t> &shape) const override;
+
+ private:
+  std::shared_ptr<serving::InferSession> session_{nullptr};
+  uint32_t model_id_ = 0;
+};
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_MODEL_H

mindspore_serving/ccsrc/worker/notfiy_master/base_notify.h

@@ -0,0 +1,38 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_BASE_NOTIFY_H
+#define MINDSPORE_SERVING_BASE_NOTIFY_H
+#include "common/serving_common.h"
+#include "common/servable.h"
+
+namespace mindspore {
+namespace serving {
+
+class MS_API BaseNotifyMaster {
+ public:
+  BaseNotifyMaster() = default;
+  virtual ~BaseNotifyMaster() = default;
+  virtual Status Register(const std::vector<WorkerSpec> &worker_specs) = 0;
+  virtual Status Unregister() = 0;
+  virtual Status AddWorker(const WorkerSpec &worker_spec) = 0;
+  virtual Status RemoveWorker(const WorkerSpec &worker_spec) = 0;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVING_BASE_NOTIFY_H

mindspore_serving/ccsrc/worker/notfiy_master/grpc_notfiy.cc

@@ -0,0 +1,150 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "worker/notfiy_master/grpc_notify.h"
+#include <memory>
+#include <thread>
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/health_check_service_interface.h>
+#include <grpcpp/ext/proto_server_reflection_plugin.h>
+#include "common/exit_handle.h"
+
+namespace mindspore {
+namespace serving {
+
+GrpcNotfiyMaster::GrpcNotfiyMaster(const std::string &master_ip, uint32_t master_port, const std::string &host_ip,
+                                   uint32_t host_port)
+    : master_ip_(master_ip), master_port_(master_port), host_ip_(host_ip), host_port_(host_port) {
+  master_address_ = master_ip_ + ":" + std::to_string(master_port);
+  worker_address_ = host_ip_ + ":" + std::to_string(host_port_);
+  auto channel = grpc::CreateChannel(master_address_, grpc::InsecureChannelCredentials());
+  stub_ = proto::MSMaster::NewStub(channel);
+}
+
+GrpcNotfiyMaster::~GrpcNotfiyMaster() = default;
+
+Status GrpcNotfiyMaster::Register(const std::vector<WorkerSpec> &worker_specs) {
+  const int32_t REGISTER_TIME_OUT = 60;
+  const int32_t REGISTER_INTERVAL = 1;
+  auto loop = REGISTER_TIME_OUT;
+  while (loop-- && !ExitHandle::Instance().HasStopped()) {
+    MSI_LOG(INFO) << "Register to " << master_address_;
+    proto::RegisterRequest request;
+    request.set_address(worker_address_);
+    for (size_t i = 0; i < worker_specs.size(); i++) {
+      auto &spec = worker_specs[i];
+      auto worker_spec = request.add_worker_spec();
+      worker_spec->set_name(spec.servable_name);
+      worker_spec->set_version_number(spec.version_number);
+      for (auto &method : spec.methods) {
+        auto proto_method = worker_spec->add_methods();
+        proto_method->set_name(method.name);
+        for (auto &name : method.input_names) {
+          proto_method->add_input_names(name);
+        }
+      }
+    }
+
+    proto::RegisterReply reply;
+    grpc::ClientContext context;
+    std::chrono::system_clock::time_point deadline =
+      std::chrono::system_clock::now() + std::chrono::seconds(REGISTER_INTERVAL);
+    context.set_deadline(deadline);
+    grpc::Status status = stub_->Register(&context, request, &reply);
+    if (status.ok()) {
+      MSI_LOG(INFO) << "Register SUCCESS ";
+      return SUCCESS;
+    }
+    std::this_thread::sleep_for(std::chrono::milliseconds(REGISTER_INTERVAL * 1000));
+  }
+  if (ExitHandle::Instance().HasStopped()) {
+    MSI_LOG_INFO << "Worker exit, stop registration";
+    return FAILED;
+  }
+  return INFER_STATUS_LOG_ERROR(SYSTEM_ERROR) << "Register TimeOut";
+}
+
+Status GrpcNotfiyMaster::Unregister() {
+  if (is_stoped_.load()) {
+    return SUCCESS;
+  }
+  is_stoped_ = true;
+  proto::ExitRequest request;
+  request.set_address(worker_address_);
+  proto::ExitReply reply;
+  grpc::ClientContext context;
+  const int32_t TIME_OUT = 1;
+  std::chrono::system_clock::time_point deadline = std::chrono::system_clock::now() + std::chrono::seconds(TIME_OUT);
+  context.set_deadline(deadline);
+  grpc::Status status = stub_->Exit(&context, request, &reply);
+  if (status.ok()) {
+    MSI_LOG(INFO) << "Exit SUCCESS ";
+    return SUCCESS;
+  }
+  return INFER_STATUS_LOG_ERROR(SYSTEM_ERROR) << "Exit Failed";
+}
+
+Status GrpcNotfiyMaster::AddWorker(const WorkerSpec &worker_spec) {
+  proto::AddWorkerReply reply;
+  grpc::ClientContext context;
+  proto::AddWorkerRequest request;
+  request.set_address(worker_address_);
+  request.mutable_worker_spec()->set_name(worker_spec.servable_name);
+  request.mutable_worker_spec()->set_version_number(worker_spec.version_number);
+  for (auto &method : worker_spec.methods) {
+    auto proto_method = request.mutable_worker_spec()->add_methods();
+    proto_method->set_name(method.name);
+    for (auto &name : method.input_names) {
+      proto_method->add_input_names(name);
+    }
+  }
+  const int32_t INTERVAL = 1;
+  std::chrono::system_clock::time_point deadline = std::chrono::system_clock::now() + std::chrono::seconds(INTERVAL);
+  context.set_deadline(deadline);
+  grpc::Status status = stub_->AddWorker(&context, request, &reply);
+  if (status.ok()) {
+    MSI_LOG(INFO) << "AddWorker SUCCESS ";
+    return SUCCESS;
+  }
+  return INFER_STATUS_LOG_ERROR(SYSTEM_ERROR) << "AddWorker Failed";
+}
+
+Status GrpcNotfiyMaster::RemoveWorker(const WorkerSpec &worker_spec) {
+  proto::RemoveWorkerReply reply;
+  grpc::ClientContext context;
+  proto::RemoveWorkerRequest request;
+  request.set_address(worker_address_);
+  request.mutable_worker_spec()->set_name(worker_spec.servable_name);
+  request.mutable_worker_spec()->set_version_number(worker_spec.version_number);
+  for (auto &method : worker_spec.methods) {
+    auto proto_method = request.mutable_worker_spec()->add_methods();
+    proto_method->set_name(method.name);
+    for (auto &name : method.input_names) {
+      proto_method->add_input_names(name);
+    }
+  }
+  const int32_t INTERVAL = 1;
+  std::chrono::system_clock::time_point deadline = std::chrono::system_clock::now() + std::chrono::seconds(INTERVAL);
+  context.set_deadline(deadline);
+  grpc::Status status = stub_->RemoveWorker(&context, request, &reply);
+  if (status.ok()) {
+    MSI_LOG(INFO) << "RemoveWorker SUCCESS ";
+    return SUCCESS;
+  }
+  return INFER_STATUS_LOG_ERROR(SYSTEM_ERROR) << "RemoveWorker Failed";
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/worker/notfiy_master/grpc_notify.h

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_GRPC_NOTIFY_H
+#define MINDSPORE_SERVING_GRPC_NOTIFY_H
+#include "worker/notfiy_master/base_notify.h"
+#include "proto/ms_master.pb.h"
+#include "proto/ms_master.grpc.pb.h"
+
+namespace mindspore {
+namespace serving {
+
+class MS_API GrpcNotfiyMaster : public BaseNotifyMaster {
+ public:
+  GrpcNotfiyMaster(const std::string &master_ip, uint32_t master_port, const std::string &host_ip, uint32_t host_port);
+  ~GrpcNotfiyMaster() override;
+  Status Register(const std::vector<WorkerSpec> &worker_specs) override;
+  Status Unregister() override;
+  Status AddWorker(const WorkerSpec &worker_spec) override;
+  Status RemoveWorker(const WorkerSpec &worker_spec) override;
+
+ private:
+  std::string master_ip_;
+  uint32_t master_port_;
+  std::string host_ip_;
+  uint32_t host_port_;
+  std::string worker_address_;
+  std::string master_address_;
+
+  std::unique_ptr<proto::MSMaster::Stub> stub_;
+  std::atomic<bool> is_stoped_{false};
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVING_GRPC_NOTIFY_H

mindspore_serving/ccsrc/worker/notfiy_master/local_notify.cc

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "worker/notfiy_master/local_notify.h"
+#include "master/server.h"
+namespace mindspore {
+namespace serving {
+
+Status LocalNotifyMaster::Register(const std::vector<WorkerSpec> &worker_specs) {
+  return Server::Instance().GetDispatcher()->RegisterLocalServable(worker_specs);
+}
+
+Status LocalNotifyMaster::Unregister() { return Server::Instance().GetDispatcher()->UnregisterLocalServable(); }
+
+Status LocalNotifyMaster::AddWorker(const WorkerSpec &worker_spec) {
+  return Server::Instance().GetDispatcher()->AddLocalServable(worker_spec);
+}
+
+Status LocalNotifyMaster::RemoveWorker(const WorkerSpec &worker_spec) {
+  return Server::Instance().GetDispatcher()->RemoveLocalServable(worker_spec);
+}
+
+}  // namespace serving
+}  // namespace mindspore

mindspore_serving/ccsrc/worker/notfiy_master/local_notify.h

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_LOCAL_NOTIFY_H
+#define MINDSPORE_SERVING_LOCAL_NOTIFY_H
+#include "worker/notfiy_master/base_notify.h"
+
+namespace mindspore {
+namespace serving {
+
+class MS_API LocalNotifyMaster : public BaseNotifyMaster {
+ public:
+  LocalNotifyMaster() = default;
+  ~LocalNotifyMaster() override = default;
+  Status Register(const std::vector<WorkerSpec> &worker_specs) override;
+  Status Unregister() override;
+  Status AddWorker(const WorkerSpec &worker_spec) override;
+  Status RemoveWorker(const WorkerSpec &worker_spec) override;
+};
+
+}  // namespace serving
+}  // namespace mindspore
+
+#endif  // MINDSPORE_SERVING_LOCAL_NOTIFY_H

mindspore_serving/ccsrc/worker/postprocess.cc

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "worker/postprocess.h"
+#include <utility>
+
+namespace mindspore::serving {
+
+void PostprocessStorage::Register(const std::string &postprocess_name, std::shared_ptr<PostprocessBase> postprocess) {
+  if (postprocess_map_.find(postprocess_name) != postprocess_map_.end()) {
+    MSI_LOG_WARNING << "postprocess " << postprocess_name << " has been registered";
+  }
+  postprocess_map_[postprocess_name] = std::move(postprocess);
+}
+
+std::shared_ptr<PostprocessBase> PostprocessStorage::GetPostprocess(const std::string &postprocess_name) const {
+  auto it = postprocess_map_.find(postprocess_name);
+  if (it != postprocess_map_.end()) {
+    return it->second;
+  }
+  return nullptr;
+}
+
+PostprocessStorage &PostprocessStorage::Instance() {
+  static PostprocessStorage storage;
+  return storage;
+}
+
+RegPostprocess::RegPostprocess(const std::string &postprocess_name, std::shared_ptr<PostprocessBase> postprocess) {
+  MSI_LOG_INFO << "Register C++ postprocess " << postprocess_name;
+  PostprocessStorage::Instance().Register(postprocess_name, std::move(postprocess));
+}
+
+}  // namespace mindspore::serving

mindspore_serving/ccsrc/worker/postprocess.h

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_SERVING_POSTPROCESS_H
+#define MINDSPORE_SERVING_POSTPROCESS_H
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include <string>
+
+#include "common/serving_common.h"
+#include "common/instance.h"
+
+namespace mindspore::serving {
+
+class PostprocessBase : public std::enable_shared_from_this<PostprocessBase> {
+ public:
+  PostprocessBase() = default;
+  virtual ~PostprocessBase() = default;
+
+  virtual Status Postprocess(const std::string &postprocess_name, const InstanceData &input, InstanceData &output) = 0;
+  virtual size_t GetInputsCount(const std::string &postprocess_name) const = 0;
+  virtual size_t GetOutputsCount(const std::string &postprocess_name) const = 0;
+  virtual bool IsPythonPostprocess() const { return false; }
+};
+
+class MS_API PostprocessStorage {
+ public:
+  void Register(const std::string &postprocess_name, std::shared_ptr<PostprocessBase> postprocess);
+
+  std::shared_ptr<PostprocessBase> GetPostprocess(const std::string &postprocess_name) const;
+
+  static PostprocessStorage &Instance();
+
+ private:
+  std::unordered_map<std::string, std::shared_ptr<PostprocessBase>> postprocess_map_;
+};
+
+class RegPostprocess {
+ public:
+  RegPostprocess(const std::string &postprocess_name, std::shared_ptr<PostprocessBase> postprocess);
+};
+
+#define REGISTER_POSTPROCESS(cls_name, postprocess_name) \
+  static RegPostprocess g_register_postprocess_##cls_name(postprocess_name, std::make_shared<cls_name>());
+
+}  // namespace mindspore::serving
+
+#endif  // MINDSPORE_SERVING_POSTPROCESS_H