Serving, distributed matmul st

example/matmul_distributed/agent.py

@@ -0,0 +1,33 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Start agents of Distributed Servable matmul"""
+
+from mindspore_serving.worker import distributed
+
+
+def start_agents():
+    """Start all the worker agents in current machine"""
+    model_files = []
+    group_configs = []
+    for i in range(8):
+        model_files.append(f"model/device{i}/matmul.mindir")
+        group_configs.append(f"model/device{i}/group_config.pb")
+
+    distributed.startup_worker_agents(worker_ip="127.0.0.1", worker_port=6200, model_files=model_files,
+                                      group_config_files=group_configs)
+
+
+if __name__ == '__main__':
+    start_agents()

example/matmul_distributed/client.py

@@ -0,0 +1,29 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Client for distributed matmul"""
+import numpy as np
+from mindspore_serving.client import Client
+
+
+def run_matmul():
+    """Run client of distributed matmul"""
+    client = Client("localhost", 5500, "matmul", "predict")
+    instance = {"x": np.ones((128, 96), np.float32)}
+    result = client.infer(instance)
+    print("result:\n", result)
+
+
+if __name__ == '__main__':
+    run_matmul()

example/matmul_distributed/export_model/distributed_inference.py

@@ -0,0 +1,42 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+'''distributed inference
+The sample can be run on Ascend 910 AI processor.
+'''
+import numpy as np
+from net import Net
+from mindspore import context, Model, Tensor, export
+from mindspore.communication import init
+
+
+def test_inference():
+    """distributed inference after distributed training"""
+    context.set_context(mode=context.GRAPH_MODE)
+    init(backend_name="hccl")
+    context.set_auto_parallel_context(full_batch=True, parallel_mode="semi_auto_parallel",
+                                      device_num=8, group_ckpt_save_file="./group_config.pb")
+
+    predict_data = create_predict_data()
+    network = Net(matmul_size=(96, 16))
+    model = Model(network)
+    model.infer_predict_layout(Tensor(predict_data))
+    # pylint: disable=protected-access
+    export(model._predict_network, Tensor(predict_data), file_name="matmul", file_format="MINDIR")
+
+
+def create_predict_data():
+    """user-defined predict data"""
+    inputs_np = np.random.randn(128, 96).astype(np.float32)
+    return Tensor(inputs_np)

example/matmul_distributed/export_model/export_model.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+EXEC_PATH=$(pwd)
+
+export RANK_TABLE_FILE=${EXEC_PATH}/rank_table_8pcs.json
+export RANK_SIZE=8
+
+rm -rf device*
+for ((i = 1; i < ${RANK_SIZE}; i++)); do
+  mkdir device$i
+  cp *.py ./device$i
+  cd ./device$i
+  export DEVICE_ID=$i
+  export RANK_ID=$i
+  echo "start inference for device $i"
+  pytest -sv ./distributed_inference.py::test_inference >inference.log$i 2>&1 &
+  cd ../
+done
+
+mkdir device0
+cp *.py ./device0
+cd ./device0
+export DEVICE_ID=0
+export RANK_ID=0
+echo "start inference for device 0"
+pytest -sv ./distributed_inference.py::test_inference >inference.log0 2>&1
+if [ $? -eq 0 ]; then
+  echo "inference success"
+else
+  echo "inference failed"
+  exit 2
+fi
+cd ../
+
+output_dir=../model
+rm -rf ${output_dir}/device*
+for ((i = 0; i < ${RANK_SIZE}; i++)); do
+  mkdir -p ${output_dir}/device${i}
+  cp device${i}/*.mindir ${output_dir}/device${i}/
+  cp device${i}/*.pb ${output_dir}/device${i}/
+done
+echo "copy models success"

example/matmul_distributed/export_model/net.py

@@ -0,0 +1,40 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+'''net
+The sample can be run on Ascend 910 AI processor.
+'''
+import numpy as np
+from mindspore import Tensor, Parameter, ops
+from mindspore.nn import Cell
+
+
+class Net(Cell):
+    """Net"""
+
+    def __init__(self, matmul_size, transpose_a=False, transpose_b=False, strategy=None):
+        """init"""
+        super().__init__()
+        matmul_np = np.full(matmul_size, 0.5, dtype=np.float32)
+        self.matmul_weight = Parameter(Tensor(matmul_np))
+        self.matmul = ops.MatMul(transpose_a=transpose_a, transpose_b=transpose_b)
+        self.neg = ops.Neg()
+        if strategy is not None:
+            self.matmul.shard(strategy)
+
+    def construct(self, inputs):
+        """construct"""
+        x = self.matmul(inputs, self.matmul_weight)
+        x = self.neg(x)
+        return x

example/matmul_distributed/export_model/rank_table_8pcs.json

@@ -0,0 +1,53 @@
+{
+  "version": "1.0",
+  "server_count": "1",
+  "server_list": [
+    {
+      "server_id": "127.0.0.1",
+      "device": [
+        {
+          "device_id": "0",
+          "device_ip": "192.1.27.6",
+          "rank_id": "0"
+        },
+        {
+          "device_id": "1",
+          "device_ip": "192.2.27.6",
+          "rank_id": "1"
+        },
+        {
+          "device_id": "2",
+          "device_ip": "192.3.27.6",
+          "rank_id": "2"
+        },
+        {
+          "device_id": "3",
+          "device_ip": "192.4.27.6",
+          "rank_id": "3"
+        },
+        {
+          "device_id": "4",
+          "device_ip": "192.1.27.7",
+          "rank_id": "4"
+        },
+        {
+          "device_id": "5",
+          "device_ip": "192.2.27.7",
+          "rank_id": "5"
+        },
+        {
+          "device_id": "6",
+          "device_ip": "192.3.27.7",
+          "rank_id": "6"
+        },
+        {
+          "device_id": "7",
+          "device_ip": "192.4.27.7",
+          "rank_id": "7"
+        }
+      ],
+      "host_nic_ip": "reserve"
+    }
+  ],
+  "status": "completed"
+}

example/matmul_distributed/master_with_worker.py

@@ -0,0 +1,35 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Start Distributed Servable matmul"""
+
+import os
+import sys
+from mindspore_serving import master
+from mindspore_serving.worker import distributed
+
+
+def start():
+    servable_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
+    distributed.start_distributed_servable_in_master(servable_dir, "matmul",
+                                                     rank_table_json_file="rank_table_8pcs.json",
+                                                     version_number=1,
+                                                     worker_ip="127.0.0.1", worker_port=6200)
+
+    master.start_grpc_server("127.0.0.1", 5500)
+    master.start_restful_server("127.0.0.1", 1500)
+
+
+if __name__ == "__main__":
+    start()

example/matmul_distributed/matmul/servable_config.py

@@ -0,0 +1,25 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Distributed matmul config python file"""
+from mindspore_serving.worker import distributed
+from mindspore_serving.worker import register
+
+distributed.declare_distributed_servable(rank_size=8, stage_size=1, with_batch_dim=False)
+
+
+@register.register_method(output_names=["y"])
+def predict(x):
+    y = register.call_servable(x)
+    return y

example/matmul_distributed/rank_table_8pcs.json

@@ -0,0 +1,53 @@
+{
+  "version": "1.0",
+  "server_count": "1",
+  "server_list": [
+    {
+      "server_id": "127.0.0.1",
+      "device": [
+        {
+          "device_id": "0",
+          "device_ip": "192.1.27.6",
+          "rank_id": "0"
+        },
+        {
+          "device_id": "1",
+          "device_ip": "192.2.27.6",
+          "rank_id": "1"
+        },
+        {
+          "device_id": "2",
+          "device_ip": "192.3.27.6",
+          "rank_id": "2"
+        },
+        {
+          "device_id": "3",
+          "device_ip": "192.4.27.6",
+          "rank_id": "3"
+        },
+        {
+          "device_id": "4",
+          "device_ip": "192.1.27.7",
+          "rank_id": "4"
+        },
+        {
+          "device_id": "5",
+          "device_ip": "192.2.27.7",
+          "rank_id": "5"
+        },
+        {
+          "device_id": "6",
+          "device_ip": "192.3.27.7",
+          "rank_id": "6"
+        },
+        {
+          "device_id": "7",
+          "device_ip": "192.4.27.7",
+          "rank_id": "7"
+        }
+      ],
+      "host_nic_ip": "reserve"
+    }
+  ],
+  "status": "completed"
+}