From 2a8ba0a2c1420763359be9dcb2b142a84be2e8f0 Mon Sep 17 00:00:00 2001 From: xuyongfei Date: Fri, 26 Feb 2021 15:58:33 +0800 Subject: [PATCH] Serving, distributed matmul st --- example/matmul_distributed/agent.py | 33 ++++++++++++ example/matmul_distributed/client.py | 29 ++++++++++ .../export_model/distributed_inference.py | 42 +++++++++++++++ .../export_model/export_model.sh | 42 +++++++++++++++ .../matmul_distributed/export_model/net.py | 40 ++++++++++++++ .../export_model/rank_table_8pcs.json | 53 +++++++++++++++++++ .../matmul_distributed/master_with_worker.py | 35 ++++++++++++ .../matmul/servable_config.py | 25 +++++++++ .../matmul_distributed/rank_table_8pcs.json | 53 +++++++++++++++++++ 9 files changed, 352 insertions(+) create mode 100644 example/matmul_distributed/agent.py create mode 100644 example/matmul_distributed/client.py create mode 100644 example/matmul_distributed/export_model/distributed_inference.py create mode 100644 example/matmul_distributed/export_model/export_model.sh create mode 100644 example/matmul_distributed/export_model/net.py create mode 100644 example/matmul_distributed/export_model/rank_table_8pcs.json create mode 100644 example/matmul_distributed/master_with_worker.py create mode 100644 example/matmul_distributed/matmul/servable_config.py create mode 100644 example/matmul_distributed/rank_table_8pcs.json diff --git a/example/matmul_distributed/agent.py b/example/matmul_distributed/agent.py new file mode 100644 index 0000000..f7e32a3 --- /dev/null +++ b/example/matmul_distributed/agent.py @@ -0,0 +1,33 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Start agents of Distributed Servable matmul""" + +from mindspore_serving.worker import distributed + + +def start_agents(): + """Start all the worker agents in current machine""" + model_files = [] + group_configs = [] + for i in range(8): + model_files.append(f"model/device{i}/matmul.mindir") + group_configs.append(f"model/device{i}/group_config.pb") + + distributed.startup_worker_agents(worker_ip="127.0.0.1", worker_port=6200, model_files=model_files, + group_config_files=group_configs) + + +if __name__ == '__main__': + start_agents() diff --git a/example/matmul_distributed/client.py b/example/matmul_distributed/client.py new file mode 100644 index 0000000..876e8dd --- /dev/null +++ b/example/matmul_distributed/client.py @@ -0,0 +1,29 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Client for distributed matmul""" +import numpy as np +from mindspore_serving.client import Client + + +def run_matmul(): + """Run client of distributed matmul""" + client = Client("localhost", 5500, "matmul", "predict") + instance = {"x": np.ones((128, 96), np.float32)} + result = client.infer(instance) + print("result:\n", result) + + +if __name__ == '__main__': + run_matmul() diff --git a/example/matmul_distributed/export_model/distributed_inference.py b/example/matmul_distributed/export_model/distributed_inference.py new file mode 100644 index 0000000..d9d94a4 --- /dev/null +++ b/example/matmul_distributed/export_model/distributed_inference.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +'''distributed inference +The sample can be run on Ascend 910 AI processor. +''' +import numpy as np +from net import Net +from mindspore import context, Model, Tensor, export +from mindspore.communication import init + + +def test_inference(): + """distributed inference after distributed training""" + context.set_context(mode=context.GRAPH_MODE) + init(backend_name="hccl") + context.set_auto_parallel_context(full_batch=True, parallel_mode="semi_auto_parallel", + device_num=8, group_ckpt_save_file="./group_config.pb") + + predict_data = create_predict_data() + network = Net(matmul_size=(96, 16)) + model = Model(network) + model.infer_predict_layout(Tensor(predict_data)) + # pylint: disable=protected-access + export(model._predict_network, Tensor(predict_data), file_name="matmul", file_format="MINDIR") + + +def create_predict_data(): + """user-defined predict data""" + inputs_np = np.random.randn(128, 96).astype(np.float32) + return Tensor(inputs_np) diff --git a/example/matmul_distributed/export_model/export_model.sh b/example/matmul_distributed/export_model/export_model.sh new file mode 100644 index 0000000..50769a3 --- /dev/null +++ b/example/matmul_distributed/export_model/export_model.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +EXEC_PATH=$(pwd) + +export RANK_TABLE_FILE=${EXEC_PATH}/rank_table_8pcs.json +export RANK_SIZE=8 + +rm -rf device* +for ((i = 1; i < ${RANK_SIZE}; i++)); do + mkdir device$i + cp *.py ./device$i + cd ./device$i + export DEVICE_ID=$i + export RANK_ID=$i + echo "start inference for device $i" + pytest -sv ./distributed_inference.py::test_inference >inference.log$i 2>&1 & + cd ../ +done + +mkdir device0 +cp *.py ./device0 +cd ./device0 +export DEVICE_ID=0 +export RANK_ID=0 +echo "start inference for device 0" +pytest -sv ./distributed_inference.py::test_inference >inference.log0 2>&1 +if [ $? -eq 0 ]; then + echo "inference success" +else + echo "inference failed" + exit 2 +fi +cd ../ + +output_dir=../model +rm -rf ${output_dir}/device* +for ((i = 0; i < ${RANK_SIZE}; i++)); do + mkdir -p ${output_dir}/device${i} + cp device${i}/*.mindir ${output_dir}/device${i}/ + cp device${i}/*.pb ${output_dir}/device${i}/ +done +echo "copy models success" diff --git a/example/matmul_distributed/export_model/net.py b/example/matmul_distributed/export_model/net.py new file mode 100644 index 0000000..feeb499 --- /dev/null +++ b/example/matmul_distributed/export_model/net.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +'''net +The sample can be run on Ascend 910 AI processor. +''' +import numpy as np +from mindspore import Tensor, Parameter, ops +from mindspore.nn import Cell + + +class Net(Cell): + """Net""" + + def __init__(self, matmul_size, transpose_a=False, transpose_b=False, strategy=None): + """init""" + super().__init__() + matmul_np = np.full(matmul_size, 0.5, dtype=np.float32) + self.matmul_weight = Parameter(Tensor(matmul_np)) + self.matmul = ops.MatMul(transpose_a=transpose_a, transpose_b=transpose_b) + self.neg = ops.Neg() + if strategy is not None: + self.matmul.shard(strategy) + + def construct(self, inputs): + """construct""" + x = self.matmul(inputs, self.matmul_weight) + x = self.neg(x) + return x diff --git a/example/matmul_distributed/export_model/rank_table_8pcs.json b/example/matmul_distributed/export_model/rank_table_8pcs.json new file mode 100644 index 0000000..c79e89e --- /dev/null +++ b/example/matmul_distributed/export_model/rank_table_8pcs.json @@ -0,0 +1,53 @@ +{ + "version": "1.0", + "server_count": "1", + "server_list": [ + { + "server_id": "127.0.0.1", + "device": [ + { + "device_id": "0", + "device_ip": "192.1.27.6", + "rank_id": "0" + }, + { + "device_id": "1", + "device_ip": "192.2.27.6", + "rank_id": "1" + }, + { + "device_id": "2", + "device_ip": "192.3.27.6", + "rank_id": "2" + }, + { + "device_id": "3", + "device_ip": "192.4.27.6", + "rank_id": "3" + }, + { + "device_id": "4", + "device_ip": "192.1.27.7", + "rank_id": "4" + }, + { + "device_id": "5", + "device_ip": "192.2.27.7", + "rank_id": "5" + }, + { + "device_id": "6", + "device_ip": "192.3.27.7", + "rank_id": "6" + }, + { + "device_id": "7", + "device_ip": "192.4.27.7", + "rank_id": "7" + } + ], + "host_nic_ip": "reserve" + } + ], + "status": "completed" +} diff --git a/example/matmul_distributed/master_with_worker.py b/example/matmul_distributed/master_with_worker.py new file mode 100644 index 0000000..3e28641 --- /dev/null +++ b/example/matmul_distributed/master_with_worker.py @@ -0,0 +1,35 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Start Distributed Servable matmul""" + +import os +import sys +from mindspore_serving import master +from mindspore_serving.worker import distributed + + +def start(): + servable_dir = os.path.dirname(os.path.realpath(sys.argv[0])) + distributed.start_distributed_servable_in_master(servable_dir, "matmul", + rank_table_json_file="rank_table_8pcs.json", + version_number=1, + worker_ip="127.0.0.1", worker_port=6200) + + master.start_grpc_server("127.0.0.1", 5500) + master.start_restful_server("127.0.0.1", 1500) + + +if __name__ == "__main__": + start() diff --git a/example/matmul_distributed/matmul/servable_config.py b/example/matmul_distributed/matmul/servable_config.py new file mode 100644 index 0000000..ad4fa35 --- /dev/null +++ b/example/matmul_distributed/matmul/servable_config.py @@ -0,0 +1,25 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Distributed matmul config python file""" +from mindspore_serving.worker import distributed +from mindspore_serving.worker import register + +distributed.declare_distributed_servable(rank_size=8, stage_size=1, with_batch_dim=False) + + +@register.register_method(output_names=["y"]) +def predict(x): + y = register.call_servable(x) + return y diff --git a/example/matmul_distributed/rank_table_8pcs.json b/example/matmul_distributed/rank_table_8pcs.json new file mode 100644 index 0000000..c79e89e --- /dev/null +++ b/example/matmul_distributed/rank_table_8pcs.json @@ -0,0 +1,53 @@ +{ + "version": "1.0", + "server_count": "1", + "server_list": [ + { + "server_id": "127.0.0.1", + "device": [ + { + "device_id": "0", + "device_ip": "192.1.27.6", + "rank_id": "0" + }, + { + "device_id": "1", + "device_ip": "192.2.27.6", + "rank_id": "1" + }, + { + "device_id": "2", + "device_ip": "192.3.27.6", + "rank_id": "2" + }, + { + "device_id": "3", + "device_ip": "192.4.27.6", + "rank_id": "3" + }, + { + "device_id": "4", + "device_ip": "192.1.27.7", + "rank_id": "4" + }, + { + "device_id": "5", + "device_ip": "192.2.27.7", + "rank_id": "5" + }, + { + "device_id": "6", + "device_ip": "192.3.27.7", + "rank_id": "6" + }, + { + "device_id": "7", + "device_ip": "192.4.27.7", + "rank_id": "7" + } + ], + "host_nic_ip": "reserve" + } + ], + "status": "completed" +}