From c53be79d4d7a61e904db8b146cbf626776e42785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cxujincai=E2=80=9D?= <“xujincai@huawei.com”> Date: Fri, 5 Mar 2021 16:08:07 +0800 Subject: [PATCH] st for matmul_distribute --- tests/st/matmul_distributed/__init__.py | 14 ++ .../matmul_distributed/matmul_distribute.sh | 220 ++++++++++++++++++ .../test_matmul_distribute.py | 42 ++++ 3 files changed, 276 insertions(+) create mode 100644 tests/st/matmul_distributed/__init__.py create mode 100644 tests/st/matmul_distributed/matmul_distribute.sh create mode 100644 tests/st/matmul_distributed/test_matmul_distribute.py diff --git a/tests/st/matmul_distributed/__init__.py b/tests/st/matmul_distributed/__init__.py new file mode 100644 index 0000000..919b057 --- /dev/null +++ b/tests/st/matmul_distributed/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/matmul_distributed/matmul_distribute.sh b/tests/st/matmul_distributed/matmul_distribute.sh new file mode 100644 index 0000000..82f4c2a --- /dev/null +++ b/tests/st/matmul_distributed/matmul_distribute.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +export GLOG_v=1 +export DEVICE_ID=1 + +MINDSPORE_INSTALL_PATH=$1 +ENV_DEVICE_ID=$DEVICE_ID +CURRPATH=$(cd "$(dirname $0)" || exit; pwd) +CURRUSER=$(whoami) +PROJECT_PATH=${CURRPATH}/../../../ +echo "MINDSPORE_INSTALL_PATH:" ${MINDSPORE_INSTALL_PATH} +echo "ENV_DEVICE_ID:" ${ENV_DEVICE_ID} +echo "CURRPATH:" ${CURRPATH} +echo "CURRUSER:" ${CURRUSER} +echo "PROJECT_PATH:" ${PROJECT_PATH} + +export LD_LIBRARY_PATH=${MINDSPORE_INSTALL_PATH}/lib:${LD_LIBRARY_PATH} +#export PYTHONPATH=${MINDSPORE_INSTALL_PATH}/:${PYTHONPATH} + +echo "LD_LIBRARY_PATH: " ${LD_LIBRARY_PATH} +echo "PYTHONPATH: " ${PYTHONPATH} +echo "-------------show MINDSPORE_INSTALL_PATH----------------" +ls -l ${MINDSPORE_INSTALL_PATH} +echo "------------------show /usr/lib64/----------------------" +ls -l /usr/local/python/python375/lib/ + +clean_master_pid() +{ + ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15 + if [ $? -ne 0 ] + then + echo "clean master pip failed" + fi + sleep 6 + ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'master.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 + echo "### master pid exist, clean master pip failed ###" & exit 1 + fi + ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 + echo "### master pid is killed but worker pid exist ###" & exit 1 + fi + fi +} + +clean_worker_pid() +{ + ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15 + if [ $? -ne 0 ] + then + echo "clean worker pip failed" + fi + sleep 6 + ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 + echo "### worker pid exist, clean worker pip failed ###" & exit 1 + fi + ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 + echo "### worker pid is killed but agent pid exist ###" & exit 1 + fi + fi +} + +clean_agent_pid() +{ + ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep + if [ $? -eq 0 ] + then + ps aux | grep 'agent.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 + if [ $? -eq 0 ] + then + echo "clean agent pid failed" + fi + fi +} + +prepare_model() +{ + echo "### begin to generate mode for serving matmul distribute test ###" + cd export_model + bash export_model.sh &> export_model.log + if [ $? -ne 0 ] + then + cat export_model.log + echo "### generate model for serving matmul distribute test failed ###" && exit 1 + clean_master_pid + clean_worker_pid + clean_agent_pid + cd - + fi + cd - +} + +start_service() +{ + echo "### start serving service ###" + unset http_proxy https_proxy + python3 master.py > start_master.log 2>&1 & + if [ $? -ne 0 ] + then + echo "master server failed to start." && exit 1 + fi + + result=`grep -E 'Serving gRPC server start success, listening on 127.0.0.1:5500' start_master.log | wc -l` + count=0 + while [[ ${result} -ne 1 && ${count} -lt 50 ]] + do + sleep 1 + count=$(($count+1)) + result=`grep -E 'Serving gRPC server start success, listening on 127.0.0.1:5500' start_master.log | wc -l` + done + + if [ ${count} -eq 50 ] + then + clean_master_pid + cat start_master.log + echo "start serving service failed!" && exit 1 + fi + + echo "### start serving master service end ###" + + python3 worker.py > start_worker.log 2>&1 & + if [ $? -ne 0 ] + then + echo "worker server failed to start." && exit 1 + fi + + result=`grep -E 'gRPC server start success, listening on 127.0.0.1:6200' start_worker.log | wc -l` + count=0 + while [[ ${result} -ne 1 && ${count} -lt 50 ]] + do + sleep 1 + count=$(($count+1)) + result=`grep -E 'gRPC server start success, listening on 127.0.0.1:6200' start_worker.log | wc -l` + done + + if [ ${count} -eq 50 ] + then + clean_master_pid + clean_worker_pid + cat start_worker.log + echo "start worker service failed!" && exit 1 + fi + + echo "### start worker service end ###" + + python3 agent.py > start_agent.log 2>&1 & + if [ $? -ne 0 ] + then + echo "agent server failed to start." && exit 1 + fi + + result=`grep -E 'Agent server start success, listening on 127.0.0.1:' start_agent.log | grep -E '7000|7001|7002|7003|7004|7005|7006|7007'| wc -l` + count=0 + while [[ ${result} -ne 8 && ${count} -lt 150 ]] + do + sleep 1 + count=$(($count+1)) + result=`grep -E 'Agent server start success, listening on 127.0.0.1:' start_agent.log | grep -E '7000|7001|7002|7003|7004|7005|7006|7007'| wc -l` + done + + if [ ${count} -eq 150 ] + then + clean_master_pid + clean_worker_pid + clean_agent_pid + cat start_agent.log + echo "start agent service failed!" && exit 1 + fi + + echo "### start agent service end ###" +} + +pytest_serving() +{ + unset http_proxy https_proxy + echo "### client start ###" + python3 client.py > client.log 2>&1 + if [ $? -ne 0 ] + then + clean_master_pid + clean_worker_pid + clean_agent_pid + cat client.log + echo "client failed to start." && exit 1 + fi + echo "### client end ###" +} + +test_add_model() +{ + start_service + pytest_serving + cat client.log + clean_master_pid + clean_worker_pid + clean_agent_pid +} + +echo "-----serving start-----" +rm -rf serving *.log *.dat ${CURRPATH}/matmul ${CURRPATH}/model ${CURRPATH}/kernel_meta ${CURRPATH}/somas_meta +rm -rf client.py export_model temp_rank_table master.py worker.py agent.py master_with_worker.py rank_table_8pcs.json +cp -r ../../../example/matmul_distributed/* . +prepare_model +test_add_model diff --git a/tests/st/matmul_distributed/test_matmul_distribute.py b/tests/st/matmul_distributed/test_matmul_distribute.py new file mode 100644 index 0000000..6fdc739 --- /dev/null +++ b/tests/st/matmul_distributed/test_matmul_distribute.py @@ -0,0 +1,42 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import sys +import pytest +import numpy as np + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.env_single +def test_serving(): + """test_serving""" + sh_path = os.path.split(os.path.realpath(__file__))[0] + python_path_folders = [] + for python_path in sys.path: + if os.path.isdir(python_path): + python_path_folders += [python_path] + folders = [] + for folder in python_path_folders: + folders += [os.path.join(folder, x) for x in os.listdir(folder) \ + if os.path.isdir(os.path.join(folder, x)) and \ + '/site-packages/mindspore' in os.path.join(folder, x)] + ret = os.system(f"sh {sh_path}/matmul_distribute.sh {folders[0].split('mindspore', 1)[0] + 'mindspore'}") + assert np.allclose(ret, 0) + + +if __name__ == '__main__': + test_serving()