!177 add st for matmul distribute

From: @xujincai0453
Reviewed-by: @xu-yfei,@zhangyinxia
Signed-off-by: @xu-yfei

tests/st/matmul_distributed/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

tests/st/matmul_distributed/matmul_distribute.sh

@@ -0,0 +1,220 @@
+#!/bin/bash
+
+export GLOG_v=1
+export DEVICE_ID=1
+
+MINDSPORE_INSTALL_PATH=$1
+ENV_DEVICE_ID=$DEVICE_ID
+CURRPATH=$(cd "$(dirname $0)" || exit; pwd)
+CURRUSER=$(whoami)
+PROJECT_PATH=${CURRPATH}/../../../
+echo "MINDSPORE_INSTALL_PATH:"  ${MINDSPORE_INSTALL_PATH}
+echo "ENV_DEVICE_ID:" ${ENV_DEVICE_ID}
+echo "CURRPATH:"  ${CURRPATH}
+echo "CURRUSER:"  ${CURRUSER}
+echo "PROJECT_PATH:"  ${PROJECT_PATH}
+
+export LD_LIBRARY_PATH=${MINDSPORE_INSTALL_PATH}/lib:${LD_LIBRARY_PATH}
+#export PYTHONPATH=${MINDSPORE_INSTALL_PATH}/:${PYTHONPATH}
+
+echo "LD_LIBRARY_PATH: " ${LD_LIBRARY_PATH}
+echo "PYTHONPATH: " ${PYTHONPATH}
+echo "-------------show MINDSPORE_INSTALL_PATH----------------"
+ls -l ${MINDSPORE_INSTALL_PATH}
+echo "------------------show /usr/lib64/----------------------"
+ls -l /usr/local/python/python375/lib/
+
+clean_master_pid()
+{
+  ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep
+  if [ $? -eq 0 ]
+  then
+    ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15
+    if [ $? -ne 0 ]
+    then
+      echo "clean master pip failed"
+    fi
+    sleep 6
+    ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep
+    if [ $? -eq 0 ]
+    then
+      ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9
+      echo "### master pid exist, clean master pip failed ###" & exit 1
+    fi
+    ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep
+    if [ $? -eq 0 ]
+    then
+      ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9
+      echo "### master pid is killed but worker pid exist ###" & exit 1
+    fi
+  fi
+}
+
+clean_worker_pid()
+{
+  ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep
+  if [ $? -eq 0 ]
+  then
+    ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15
+    if [ $? -ne 0 ]
+    then
+      echo "clean worker pip failed"
+    fi
+    sleep 6
+    ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep
+    if [ $? -eq 0 ]
+    then
+      ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9
+      echo "### worker pid exist, clean worker pip failed ###" & exit 1
+    fi
+    ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep
+    if [ $? -eq 0 ]
+    then
+      ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9
+      echo "### worker pid is killed but agent pid exist ###" & exit 1
+    fi
+  fi
+}
+
+clean_agent_pid()
+{
+  ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep
+  if [ $? -eq 0 ]
+  then
+    ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9
+    if [ $? -eq 0 ]
+    then
+      echo "clean agent pid failed"
+    fi
+  fi
+}
+
+prepare_model()
+{
+  echo "### begin to generate mode for serving matmul distribute test ###"
+  cd export_model
+  bash export_model.sh &> export_model.log
+  if [ $? -ne 0 ]
+  then
+    cat export_model.log
+    echo "### generate model for serving matmul distribute test failed ###" && exit 1
+    clean_master_pid
+    clean_worker_pid
+    clean_agent_pid
+    cd -
+  fi
+  cd -
+}
+
+start_service()
+{
+  echo "### start serving service ###"
+  unset http_proxy https_proxy
+  python3 master.py > start_master.log 2>&1 &
+  if [ $? -ne 0 ]
+  then
+    echo "master server failed to start." && exit 1
+  fi
+
+  result=`grep -E 'Serving gRPC server start success, listening on 127.0.0.1:5500' start_master.log | wc -l`
+  count=0
+  while [[ ${result} -ne 1 && ${count} -lt 50 ]]
+  do
+    sleep 1
+    count=$(($count+1))
+    result=`grep -E 'Serving gRPC server start success, listening on 127.0.0.1:5500' start_master.log | wc -l`
+  done
+
+  if [ ${count} -eq 50 ]
+  then
+    clean_master_pid
+    cat start_master.log
+    echo "start serving service failed!" && exit 1
+  fi
+
+  echo "### start serving master service end ###"
+
+  python3 worker.py > start_worker.log 2>&1 &
+  if [ $? -ne 0 ]
+  then
+    echo "worker server failed to start." && exit 1
+  fi
+
+  result=`grep -E 'gRPC server start success, listening on 127.0.0.1:6200' start_worker.log | wc -l`
+  count=0
+  while [[ ${result} -ne 1 && ${count} -lt 50 ]]
+  do
+    sleep 1
+    count=$(($count+1))
+    result=`grep -E 'gRPC server start success, listening on 127.0.0.1:6200' start_worker.log | wc -l`
+  done
+
+  if [ ${count} -eq 50 ]
+  then
+    clean_master_pid
+    clean_worker_pid
+    cat start_worker.log
+    echo "start worker service failed!" && exit 1
+  fi
+
+  echo "### start worker service end ###"
+
+    python3 agent.py > start_agent.log 2>&1 &
+  if [ $? -ne 0 ]
+  then
+    echo "agent server failed to start." && exit 1
+  fi
+
+  result=`grep -E 'Agent server start success, listening on 127.0.0.1:' start_agent.log | grep -E '7000|7001|7002|7003|7004|7005|7006|7007'| wc -l`
+  count=0
+  while [[ ${result} -ne 8 && ${count} -lt 150 ]]
+  do
+    sleep 1
+    count=$(($count+1))
+    result=`grep -E 'Agent server start success, listening on 127.0.0.1:' start_agent.log | grep -E '7000|7001|7002|7003|7004|7005|7006|7007'| wc -l`
+  done
+
+  if [ ${count} -eq 150 ]
+  then
+    clean_master_pid
+    clean_worker_pid
+    clean_agent_pid
+    cat start_agent.log
+    echo "start agent service failed!" && exit 1
+  fi
+
+  echo "### start agent service end ###"
+}
+
+pytest_serving()
+{
+  unset http_proxy https_proxy
+  echo "###  client start ###"
+  python3  client.py > client.log 2>&1
+  if [ $? -ne 0 ]
+  then
+    clean_master_pid
+    clean_worker_pid
+    clean_agent_pid
+    cat client.log
+    echo "client failed to start." && exit 1
+  fi
+  echo "### client end ###"
+}
+
+test_add_model()
+{
+  start_service
+  pytest_serving
+  cat client.log
+  clean_master_pid
+  clean_worker_pid
+  clean_agent_pid
+}
+
+echo "-----serving start-----"
+rm -rf serving *.log *.dat ${CURRPATH}/matmul ${CURRPATH}/model ${CURRPATH}/kernel_meta ${CURRPATH}/somas_meta
+rm -rf client.py  export_model temp_rank_table master.py worker.py agent.py master_with_worker.py  rank_table_8pcs.json
+cp -r ../../../example/matmul_distributed/* .
+prepare_model
+test_add_model

tests/st/matmul_distributed/test_matmul_distribute.py

@@ -0,0 +1,42 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+import os
+import sys
+import pytest
+import numpy as np
+
+
+@pytest.mark.level0
+@pytest.mark.platform_arm_ascend_training
+@pytest.mark.env_single
+def test_serving():
+    """test_serving"""
+    sh_path = os.path.split(os.path.realpath(__file__))[0]
+    python_path_folders = []
+    for python_path in sys.path:
+        if os.path.isdir(python_path):
+            python_path_folders += [python_path]
+    folders = []
+    for folder in python_path_folders:
+        folders += [os.path.join(folder, x) for x in os.listdir(folder) \
+                    if os.path.isdir(os.path.join(folder, x)) and \
+                    '/site-packages/mindspore' in os.path.join(folder, x)]
+    ret = os.system(f"sh {sh_path}/matmul_distribute.sh {folders[0].split('mindspore', 1)[0] + 'mindspore'}")
+    assert np.allclose(ret, 0)
+
+
+if __name__ == '__main__':
+    test_serving()