Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 668618c01b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '668618c01b'
${ noResults }
mindspore/mindspore/lite/test/run_net_train.sh

407 lines
18 kB
Raw Blame History 

  1. #!/bin/bash

  2. 

  3. # Run Export on x86 platform and create output test files:

  4. function Run_Export(){

  5.     cd $models_path || exit 1

  6.     if [[ -z "${CLOUD_MODEL_ZOO}" ]]; then

  7.         echo "CLOUD_MODEL_ZOO is not defined - exiting export models"

  8.         exit 1

  9.     fi    

  10.     # Export mindspore train models:

  11.     while read line; do

  12.         model_name=${line}

  13.         if [[ $model_name == \#* ]]; then

  14.           continue

  15.         fi

  16.         echo ${model_name}'_train_export.py' >> "${export_log_file}"

  17.         echo 'exporting' ${model_name}

  18.         echo 'docker run --user $(id -u):$(id -g) --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore_dev:5 python '${models_path}'/'${model_name}'_train_export.py' >>  "${export_log_file}"

  19.         docker run --user $(id -u):$(id -g) --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore_dev:5 python ${models_path}'/'${model_name}_train_export.py

  20.         if [ $? = 0 ]; then

  21.             export_result='export mindspore '${model_name}'_train_export pass';echo ${export_result} >> ${export_result_file}

  22.         else

  23.             export_result='export mindspore '${model_name}'_train_export failed';echo ${export_result} >> ${export_result_file}

  24.         fi

  25.     done < ${models_mindspore_train_config}

  26. }

  27. 

  28. # Run converter on x86 platform:

  29. function Run_Converter() {

  30.     # Unzip x86 runtime and convertor

  31.     cd ${x86_path} || exit 1

  32.     tar -zxf mindspore-lite-${version}-runtime-x86-${process_unit_x86}-train.tar.gz || exit 1

  33. 

  34.     tar -zxf mindspore-lite-${version}-converter-ubuntu-train.tar.gz || exit 1

  35.     cd ${x86_path}/mindspore-lite-${version}-converter-ubuntu-train || exit 1

  36.     cp converter/converter_lite ./ || exit 1

  37.     

  38. 

  39.     # Convert the models

  40.     cd ${x86_path}/mindspore-lite-${version}-converter-ubuntu-train || exit 1

  41. 

  42.     rm -rf ${ms_models_path}

  43.     mkdir -p ${ms_models_path}

  44. 

  45.     # Convert mindspore train models:

  46.     while read line; do

  47.         model_name=${line}

  48.         if [[ $model_name == \#* ]]; then

  49.           continue

  50.         fi

  51.         echo ${model_name}'_train' >> "${run_converter_log_file}"

  52.         echo './converter_lite  --fmk=MINDIR --modelFile='${models_path}'/'${model_name}'_train.mindir --outputFile='${ms_models_path}'/'${model_name}'_train  --trainModel=true' >> "${run_converter_log_file}"

  53.         LD_LIBRARY_PATH=./lib/:./third_party/protobuf/lib:./third_party/flatbuffers/lib:./third_party/glog/lib \

  54.          ./converter_lite  --fmk=MINDIR --modelFile=${models_path}/${model_name}_train.mindir \

  55.          --outputFile=${ms_models_path}/${model_name}'_train' \

  56.          --trainModel=true

  57.         if [ $? = 0 ]; then

  58.             converter_result='converter mindspore '${model_name}'_train pass';echo ${converter_result} >> ${run_converter_result_file}

  59.         else

  60.             converter_result='converter mindspore '${model_name}'_train failed';echo ${converter_result} >> ${run_converter_result_file}

  61.         fi

  62.     done < ${models_mindspore_train_config}

  63. }

  64. 

  65. # Run on x86 platform:

  66. function Run_x86() {

  67.     # Run mindspore converted train models:

  68.     while read line; do

  69.         model_name=${line}

  70.         if [[ $model_name == \#* ]]; then

  71.           continue

  72.         fi

  73.         

  74.         echo ${model_name}'_train' >> "${run_x86_log_file}"

  75.         echo 'cd  '${x86_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86}-train >> "${run_x86_log_file}"

  76.         cd ${x86_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86}-train || return 1

  77.         echo 'LD_LIBRARY_PATH='${LD_LIBRARY_PATH}':./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./net_train/net_train --modelFile='${ms_models_path}'/'${model_name}'_train.ms --inDataFile='${input_path}'/'${model_name}'_input1.bin,'${train_io_path}'/'${model_name}'_input2.bin --expectedDataFile='${train_io_path}'/'${model_name}'_outputs.bin --exportFile='${ms_models_path}'/'${model_name}'_train_exported.ms' >> "${run_x86_log_file}"

  78.         echo '-------------------------------------------------------------------------------' >> "${run_x86_log_file}"

  79.         LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib \

  80.         ${run_valgrind}./net_train/net_train \

  81.         --modelFile=${ms_models_path}/${model_name}_train.ms \

  82.         --inDataFile=${train_io_path}/${model_name}_input1.bin,${train_io_path}/${model_name}_input2.bin \

  83.         --expectedDataFile=${train_io_path}/${model_name}_outputs.bin \

  84.         --exportFile=${ms_models_path}/${model_name}_train_exported.ms >> "${run_x86_log_file}"

  85.         if [ $? = 0 ]; then

  86.             run_result='x86: '${model_name}'_train pass'; echo ${run_result} >> ${run_net_train_result_file}

  87.         else

  88.             run_result='x86: '${model_name}'_train failed'; echo ${run_result} >> ${run_net_train_result_file}; return 1

  89.         fi

  90.     done < ${models_mindspore_train_config}

  91. }

  92. 

  93. # Run on arm platform: 

  94. # Gets a parameter - arm64/arm32

  95. function Run_arm() {

  96.     if [ "$1" == arm64 ]; then

  97.         arm_path=${arm64_path}

  98.         process_unit=${process_unit_arm64}

  99.         version_arm=${version_arm64}

  100.         run_arm_log_file=${run_arm64_log_file} 

  101.         adb_cmd_run_file=${adb_cmd_arm64_run_file} 

  102.         adb_push_log_file=${adb_push_arm64_log_file} 

  103.         adb_cmd_file=${adb_cmd_arm64_file}

  104.     elif [ "$1" == arm32 ]; then

  105.         arm_path=${arm32_path}

  106.         process_unit=${process_unit_arm32}

  107.         version_arm=${version_arm32}

  108.         run_arm_log_file=${run_arm32_log_file} 

  109.         adb_cmd_run_file=${adb_cmd_arm32_run_file} 

  110.         adb_push_log_file=${adb_push_arm32_log_file} 

  111.         adb_cmd_file=${adb_cmd_arm32_file}

  112.     else

  113.         echo 'type ' $1 'is not supported'

  114.         exit 1

  115.     fi   

  116.     arm_type=$1

  117. 

  118.     # Unzip

  119.     cd ${arm_path} || exit 1

  120.     tar -zxf mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train.tar.gz || exit 1

  121. 

  122.     # If build with minddata, copy the minddata related libs

  123.     cd ${net_train_test_path} || exit 1

  124.     if [ -f ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/lib/libminddata-lite.so ]; then

  125.         cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/third_party/libjpeg-turbo/lib/libjpeg.so ${net_train_test_path}/libjpeg.so || exit 1

  126.         cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/third_party/libjpeg-turbo/lib/libturbojpeg.so ${net_train_test_path}/libturbojpeg.so || exit 1

  127.         cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/third_party/opencv/lib/libopencv_core.so ${net_train_test_path}/libopencv_core.so || exit 1

  128.         cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/third_party/opencv/lib/libopencv_imgcodecs.so ${net_train_test_path}/libopencv_imgcodecs.so || exit 1

  129.         cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/third_party/opencv/lib/libopencv_imgproc.so ${net_train_test_path}/libopencv_imgproc.so || exit 1

  130.         cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/lib/libminddata-lite.so ${net_train_test_path}/libminddata-lite.so || exit 1

  131.     fi

  132. 

  133.     cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/lib/libmindspore-lite.so ${net_train_test_path}/libmindspore-lite.so || exit 1

  134. #    if [ "$1" == arm64 ]; then

  135. #        cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/lib/libmindspore-lite-fp16.so ${net_train_test_path}/libmindspore-lite-fp16.so || exit 1

  136. #        cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/lib/libmindspore-lite-optimize.so ${net_train_test_path}/libmindspore-lite-optimize.so || exit 1

  137. #    fi

  138.     cp -a ${arm_path}/mindspore-lite-${version_arm}-runtime-${arm_type}-${process_unit}-train/net_train/net_train ${net_train_test_path}/net_train || exit 1

  139. 

  140.     # adb push all needed files to the phone

  141.     adb -s ${device_id} push ${net_train_test_path} /data/local/tmp/ > ${adb_push_log_file}

  142. 

  143.     # run adb ,run session ,check the result:

  144.     echo 'cd  /data/local/tmp/net_train_test' > ${adb_cmd_file}

  145.     echo 'chmod 777 net_train' >> ${adb_cmd_file}

  146. 

  147.     adb -s ${device_id} shell < ${adb_cmd_file}

  148.     

  149.     # Run mindir converted train models:

  150.     while read line; do

  151.         model_name=${line}

  152.         if [[ $model_name == \#* ]]; then

  153.           continue

  154.         fi

  155. 

  156.         # run net_train test without clib data

  157.         echo ${model_name}'_train' >> "${run_arm_log_file}"

  158.         adb -s ${device_id} push ${train_io_path}/${model_name}_input*.bin ${train_io_path}/${model_name}_outputs.bin  /data/local/tmp/net_train_test >> ${adb_push_log_file}

  159.         echo 'cd /data/local/tmp/net_train_test' > ${adb_cmd_run_file}

  160.         echo 'chmod 777 net_train' >> ${adb_cmd_run_file}

  161.         if [ "$1" == arm64 ]; then

  162.             echo 'cp  /data/local/tmp/libc++_shared.so ./' >> ${adb_cmd_run_file}

  163.             echo 'export LD_LIBRARY_PATH=/data/local/tmp/net_train_test;./net_train --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin --exportFile='${model_name}'_train_exported.ms' >> "${run_arm_log_file}"

  164.             echo 'export LD_LIBRARY_PATH=/data/local/tmp/net_train_test;./net_train --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin --exportFile='${model_name}'_train_exported.ms' >> "${adb_cmd_run_file}"

  165.         elif [ "$1" == arm32 ]; then

  166.             echo 'cp  /data/local/tmp/arm32/libc++_shared.so ./' >> ${adb_cmd_run_file}

  167.             echo 'export LD_LIBRARY_PATH=/data/local/tmp/net_train_test;./net_train --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin --exportFile='${model_name}'_train_exported.ms' >> "${run_arm_log_file}"

  168.             echo 'export LD_LIBRARY_PATH=/data/local/tmp/net_train_test;./net_train --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin --exportFile='${model_name}'_train_exported.ms' >> "${adb_cmd_run_file}"

  169.         fi    

  170. 

  171.         adb -s ${device_id} shell < ${adb_cmd_run_file} >> ${run_arm_log_file}

  172.         # TODO: change to arm_type

  173.         if [ $? = 0 ]; then

  174.             run_result=$1': '${model_name}'_train pass'; echo ${run_result} >> ${run_net_train_result_file}

  175.         else

  176.             run_result=$1': '${model_name}'_train failed'; echo ${run_result} >> ${run_net_train_result_file}; return 1

  177.         fi

  178.     done < ${models_mindspore_train_config}

  179. }

  180. 

  181. # Print start msg before run testcase

  182. function MS_PRINT_TESTCASE_START_MSG() {

  183.     echo ""

  184.     echo -e "-----------------------------------------------------------------------------------------------------------------------------------"

  185.     echo -e "env                  Testcase                                                                                             Result   "

  186.     echo -e "---                  --------                                                                                             ------   "

  187. }

  188. 

  189. # Print start msg after run testcase

  190. function MS_PRINT_TESTCASE_END_MSG() {

  191.     echo -e "-----------------------------------------------------------------------------------------------------------------------------------"

  192. }

  193. 

  194. function Print_Result() {

  195.     MS_PRINT_TESTCASE_END_MSG

  196.     while read line; do

  197.         arr=("${line}")

  198.         printf "%-15s %-20s %-90s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]} ${arr[3]}

  199.     done < $1

  200.     MS_PRINT_TESTCASE_END_MSG

  201. }

  202. 

  203. basepath=$(pwd)

  204. echo ${basepath}

  205. 

  206. train_io_path=""

  207. # Example:run_net_train.sh -r /home/emir/Work/TestingEnv/release -m /home/emir/Work/TestingEnv/train_models -i /home/emir/Work/TestingEnv/train_io -d "8KE5T19620002408"

  208. # For running on arm64, use -t to set platform tools path (for using adb commands)

  209. while getopts "r:m:d:i:e:v" opt; do

  210.     case ${opt} in

  211.         r)

  212. 	    release_path=${OPTARG}

  213.             echo "release_path is ${OPTARG}"

  214.             ;;

  215.         m)

  216. 	    models_path=${OPTARG}"/models_train"

  217.             echo "models_path is ${OPTARG}"

  218.             ;;

  219.         i)

  220. 	    train_io_path=${OPTARG}

  221.             echo "train_io_path is ${OPTARG}"

  222.             ;;

  223.         d)

  224. 	    device_id=${OPTARG}

  225.             echo "device_id is ${OPTARG}"

  226.             ;;

  227.         e)

  228. 	    enable_export=${OPTARG}

  229.             echo "enable_export = ${OPTARG}"

  230.             ;;          

  231.         v)

  232. 	    run_valgrind="valgrind "

  233.             echo "Run x86 with valgrind"

  234.             ;;            

  235.         ?)

  236.             echo "unknown para"

  237.             exit 1;;

  238.     esac

  239. done

  240. 

  241. if [[ $train_io_path == "" ]]

  242. then

  243.   train_io_path=${models_path}/input_output

  244. fi

  245. 

  246. arm64_path=${release_path}/android_aarch64

  247. file=$(ls ${arm64_path}/*runtime-arm64*train.tar.gz)

  248. file_name="${file##*/}"

  249. IFS="-" read -r -a file_name_array <<< "$file_name"

  250. version_arm64=${file_name_array[2]}

  251. process_unit_arm64=${file_name_array[5]}

  252. 

  253. arm32_path=${release_path}/android_aarch32

  254. file=$(ls ${arm32_path}/*runtime-arm32*train.tar.gz)

  255. file_name="${file##*/}"

  256. IFS="-" read -r -a file_name_array <<< "$file_name"

  257. version_arm32=${file_name_array[2]}

  258. process_unit_arm32=${file_name_array[5]}

  259. 

  260. x86_path=${release_path}/ubuntu_x86

  261. file=$(ls ${x86_path}/*runtime-x86*train.tar.gz)

  262. file_name="${file##*/}"

  263. IFS="-" read -r -a file_name_array <<< "$file_name"

  264. version=${file_name_array[2]}

  265. process_unit_x86=${file_name_array[5]}

  266. 

  267. # Set models config filepath

  268. models_mindspore_train_config=${basepath}/models_ms_train.cfg

  269. 

  270. ms_models_path=${basepath}/ms_models_train

  271. 

  272. logs_path=${basepath}/logs_train

  273. rm -rf ${logs_path}

  274. mkdir -p ${logs_path}

  275. 

  276. # Export model if enabled

  277. if [[ $enable_export == 1 ]]; then

  278.     echo "Start Exporting models ..."

  279.     # Write export result to temp file

  280.     export_log_file=${logs_path}/export_log.txt

  281.     echo ' ' > ${export_log_file}

  282. 

  283.     export_result_file=${logs_path}/export_result.txt

  284.     echo ' ' > ${export_result_file}

  285.     # Run export

  286.     Run_Export

  287.     Print_Result ${export_result_file}

  288. 

  289. fi    

  290. 

  291. # Write converter result to temp file

  292. run_converter_log_file=${logs_path}/run_converter_log.txt

  293. echo ' ' > ${run_converter_log_file}

  294. 

  295. run_converter_result_file=${logs_path}/run_converter_result.txt

  296. echo ' ' > ${run_converter_result_file}

  297. 

  298. START=$(date +%s.%N)

  299. 

  300. # Run converter

  301. echo "start run converter ..."

  302. Run_Converter

  303. Run_converter_PID=$!

  304. sleep 1

  305. 

  306. wait ${Run_converter_PID}

  307. Run_converter_status=$?

  308. 

  309. # Check converter result and return value

  310. if [[ ${Run_converter_status} = 0 ]];then

  311.     echo "Run converter success"

  312.     Print_Result ${run_converter_result_file}

  313. else

  314.     echo "Run converter failed"

  315.     cat ${run_converter_log_file}

  316.     Print_Result ${run_converter_result_file}

  317.     exit 1

  318. fi

  319. 

  320. 

  321. # Write net_train result to temp file

  322. run_net_train_result_file=${logs_path}/run_net_train_result.txt

  323. echo ' ' > ${run_net_train_result_file}

  324. 

  325. # Create log files

  326. run_x86_log_file=${logs_path}/run_x86_log.txt

  327. echo 'run x86 logs: ' > ${run_x86_log_file}

  328. 

  329. run_arm64_log_file=${logs_path}/run_arm64_log.txt

  330. echo 'run arm64 logs: ' > ${run_arm64_log_file}

  331. adb_push_arm64_log_file=${logs_path}/adb_push_arm64_log.txt

  332. adb_cmd_arm64_file=${logs_path}/adb_arm64_cmd.txt

  333. adb_cmd_arm64_run_file=${logs_path}/adb_arm64_cmd_run.txt

  334. 

  335. run_arm32_log_file=${logs_path}/run_arm32_log.txt

  336. echo 'run arm32 logs: ' > ${run_arm64_log_file}

  337. adb_push_arm32_log_file=${logs_path}/adb_push_arm32_log.txt

  338. adb_cmd_arm32_file=${logs_path}/adb_arm32_cmd.txt

  339. adb_cmd_arm32_run_file=${logs_path}/adb_arm32_cmd_run.txt

  340. 

  341. # Copy the MindSpore models:

  342. echo "Push files to net_train_test folder and run net_train"

  343. net_train_test_path=${basepath}/net_train_test

  344. rm -rf ${net_train_test_path}

  345. mkdir -p ${net_train_test_path}

  346. cp -a ${ms_models_path}/*.ms ${net_train_test_path} || exit 1

  347. 

  348. # Run on x86

  349. echo "start Run x86 ..."

  350. Run_x86 &

  351. Run_x86_PID=$!

  352. sleep 1

  353. 

  354. # wait ${Run_x86_PID}

  355. cat ${run_net_train_result_file}

  356. wait ${Run_x86_PID}

  357. Run_x86_status=$?

  358. # exit 0

  359. 

  360. # Run on arm64

  361. echo "start Run arm64 ..."

  362. Run_arm arm64 

  363. Run_arm64_status=$?

  364. sleep 3

  365. 

  366. # Run on arm32

  367. echo "start Run arm32 ..."

  368. Run_arm arm32

  369. Run_arm32_status=$?

  370. sleep 1

  371. 

  372. END=$(date +%s.%N)

  373. DIFF=$(echo "$END - $START" | bc)

  374. 

  375. function Print_Benchmark_Result() {

  376.     MS_PRINT_TESTCASE_START_MSG

  377.     while read line; do

  378.         arr=("${line}")

  379.         printf "%-20s %-100s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]}

  380.     done < ${run_net_train_result_file}

  381.     MS_PRINT_TESTCASE_END_MSG

  382. }

  383. 

  384. # Check net_train result and return value

  385. if [[ ${Run_x86_status} != 0 ]];then

  386.     echo "Run_x86 failed"

  387.     cat ${run_x86_log_file}

  388.     exit 1

  389. fi

  390. 

  391. if [[ ${Run_arm64_status} != 0 ]];then

  392.     echo "Run_arm64 failed"

  393.     cat ${run_arm64_log_file}

  394.     exit 1

  395. fi

  396. 

  397. if [[ ${Run_arm32_status} != 0 ]];then

  398.     echo "Run_arm32 failed"

  399.     cat ${run_arm32_log_file}

  400.     exit 1

  401. fi

  402. 

  403. echo "Test ended - Results:"

  404. Print_Benchmark_Result

  405. echo "Test run Time:" $DIFF

  406. exit 0