Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: a80b3c3005
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a80b3c3005'
${ noResults }
mindspore/mindspore/lite/test/run_benchmark_nets.sh

612 lines
36 kB
Raw Blame History 

  1. #!/bin/bash

  2. 

  3. # Run on x86 platform:

  4. function Run_x86() {

  5.     # Run tflite converted models:

  6.     while read line; do

  7.         model_name=${line}

  8.         if [[ $model_name == \#* ]]; then

  9.           continue

  10.         fi

  11.         echo ${model_name} >> "${run_benchmark_log_file}"

  12.         echo 'cd  '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "{run_benchmark_log_file}"

  13.         cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1

  14.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  15.         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}"

  16.         if [ $? = 0 ]; then

  17. 	    run_result='x86: '${model_name}' pass'

  18. 	    echo ${run_result} >> ${run_benchmark_result_file}

  19. 	else

  20. 	    run_result='x86: '${model_name}' failed'

  21. 	    echo ${run_result} >> ${run_benchmark_result_file}

  22. 	    return 1

  23.         fi

  24.     done < ${models_tflite_config}

  25. 

  26.     # Run caffe converted models:

  27.     while read line; do

  28.         model_name=${line}

  29.         if [[ $model_name == \#* ]]; then

  30.           continue

  31.         fi

  32.         echo ${model_name} >> "${run_benchmark_log_file}"

  33.         echo 'cd  '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}"

  34.         cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1

  35.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  36.         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}"

  37.         if [ $? = 0 ]; then

  38.             run_result='x86: '${model_name}' pass'

  39.             echo ${run_result} >> ${run_benchmark_result_file}

  40.         else

  41.             run_result='x86: '${model_name}' failed'

  42.             echo ${run_result} >> ${run_benchmark_result_file}

  43.             return 1

  44.         fi

  45.     done < ${models_caffe_config}

  46. 

  47.     # Run onnx converted models:

  48.     while read line; do

  49.         model_name=${line}

  50.         if [[ $model_name == \#* ]]; then

  51.           continue

  52.         fi

  53.         echo ${model_name} >> "${run_benchmark_log_file}"

  54.         echo 'cd  '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}"

  55.         cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1

  56.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  57.         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}"

  58.         if [ $? = 0 ]; then

  59.             run_result='x86: '${model_name}' pass'

  60.             echo ${run_result} >> ${run_benchmark_result_file}

  61.         else

  62.             run_result='x86: '${model_name}' failed'

  63.             echo ${run_result} >> ${run_benchmark_result_file}

  64.             return 1

  65.         fi

  66.     done < ${models_onnx_config}

  67. 

  68.     # Run tflite post training quantization converted models:

  69.     while read line; do

  70.         model_name=${line}

  71.         if [[ $model_name == \#* ]]; then

  72.           continue

  73.         fi

  74.         echo ${model_name} >> "${run_benchmark_log_file}"

  75.         echo 'cd  '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}"

  76.         cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1

  77.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'_posttraining.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  78.         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}_posttraining.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}_posttraining.ms.out --warmUpLoopCount=1 --loopCount=1 >> "${run_benchmark_log_file}"

  79.         if [ $? = 0 ]; then

  80.             run_result='x86: '${model_name}'_posttraining pass'

  81.             echo ${run_result} >> ${run_benchmark_result_file}

  82.         else

  83.             run_result='x86: '${model_name}'_posttraining failed'

  84.             echo ${run_result} >> ${run_benchmark_result_file}

  85.             return 1

  86.         fi

  87.     done < ${models_tflite_posttraining_config}

  88. 

  89.     # Run tflite aware training quantization converted models:

  90.     while read line; do

  91.         model_name=${line}

  92.         if [[ $model_name == \#* ]]; then

  93.           continue

  94.         fi

  95.         echo ${model_name} >> "${run_benchmark_log_file}"

  96.         echo 'cd  '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}"

  97.         cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1

  98.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --numThreads=1' >> "${run_benchmark_log_file}"

  99.         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 --numThreads=1 >> "${run_benchmark_log_file}"

  100.         if [ $? = 0 ]; then

  101.             run_result='x86: '${model_name}'_awaretraining pass'

  102.             echo ${run_result} >> ${run_benchmark_result_file}

  103.         else

  104.             run_result='x86: '${model_name}'_awaretraining failed'

  105.             echo ${run_result} >> ${run_benchmark_result_file}

  106.             return 1

  107.         fi

  108.     done < ${models_tflite_awaretraining_config}

  109. 

  110.     # Run mindspore converted models:

  111.     while read line; do

  112.         model_name=${line}

  113.         if [[ $model_name == \#* ]]; then

  114.           continue

  115.         fi

  116.         echo ${model_name} >> "${run_benchmark_log_file}"

  117.         echo 'cd  '${convertor_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_benchmark_log_file}"

  118.         cd ${convertor_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1

  119.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath='${ms_models_path}'/'${model_name}'.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  120.         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelPath=${ms_models_path}/${model_name}.ms --inDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --calibDataPath=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --warmUpLoopCount=1 --loopCount=1 --accuracyThreshold=1.5 >> "${run_benchmark_log_file}"

  121.         if [ $? = 0 ]; then

  122.             run_result='x86: '${model_name}' pass'

  123.             echo ${run_result} >> ${run_benchmark_result_file}

  124.         else

  125.             run_result='x86: '${model_name}' failed'

  126.             echo ${run_result} >> ${run_benchmark_result_file}

  127.             return 1

  128.         fi

  129.     done < ${models_mindspore_config}

  130. }

  131. 

  132. # Run on arm64 platform:

  133. function Run_arm64() {

  134.     # Run tflite converted models:

  135.     while read line; do

  136.         model_name=${line}

  137.         if [[ $model_name == \#* ]]; then

  138.           continue

  139.         fi

  140.         echo ${model_name} >> "${run_benchmark_log_file}"

  141.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  142.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  143.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> adb_run_cmd.txt

  144.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  145.         if [ $? = 0 ]; then

  146.             run_result='arm64: '${model_name}' pass'

  147.             echo ${run_result} >> ${run_benchmark_result_file}

  148.         else

  149.             run_result='arm64: '${model_name}' failed'

  150.             echo ${run_result} >> ${run_benchmark_result_file}

  151.             return 1

  152.         fi

  153.         # run benchmark test without clib data

  154.         #echo ${model_name}

  155.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  156.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_benchmark_log_file}"

  157.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  158.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  159.         if [ $? = 0 ]; then

  160.             run_result='arm64: '${model_name}' pass'

  161.             echo ${run_result} >> ${run_benchmark_result_file}

  162.         else

  163.             run_result='arm64: '${model_name}' failed'

  164.             echo ${run_result} >> ${run_benchmark_result_file}

  165.             return 1

  166.         fi

  167. 	#sleep 1

  168.     done < ${models_tflite_config}

  169. 

  170.     # Run caffe converted models:

  171.     while read line; do

  172.         model_name=${line}

  173.         if [[ $model_name == \#* ]]; then

  174.           continue

  175.         fi

  176.         echo ${model_name} >> "${run_benchmark_log_file}"

  177.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  178.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  179.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> adb_run_cmd.txt

  180.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  181.         if [ $? = 0 ]; then

  182.             run_result='arm64: '${model_name}' pass'

  183.             echo ${run_result} >> ${run_benchmark_result_file}

  184.         else

  185.             run_result='arm64: '${model_name}' failed'

  186.             echo ${run_result} >> ${run_benchmark_result_file}

  187.             return 1

  188.         fi

  189.         # run benchmark test without clib data

  190.         echo ${model_name} >> "${run_benchmark_log_file}"

  191.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  192.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_benchmark_log_file}"

  193.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  194.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  195.         if [ $? = 0 ]; then

  196.             run_result='arm64: '${model_name}' pass'

  197.             echo ${run_result} >> ${run_benchmark_result_file}

  198.         else

  199.             run_result='arm64: '${model_name}' failed'

  200.             echo ${run_result} >> ${run_benchmark_result_file}

  201.             return 1

  202.         fi

  203. 	#sleep 1

  204.     done < ${models_caffe_config}

  205. 

  206.     # Run onnx converted models:

  207.     while read line; do

  208.         model_name=${line}

  209.         if [[ $model_name == \#* ]]; then

  210.           continue

  211.         fi

  212.         echo ${model_name} >> "${run_benchmark_log_file}"

  213.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  214.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  215.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> adb_run_cmd.txt

  216.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  217.         if [ $? = 0 ]; then

  218.             run_result='arm64: '${model_name}' pass'

  219.             echo ${run_result} >> ${run_benchmark_result_file}

  220.         else

  221.             run_result='arm64: '${model_name}' failed'

  222.             echo ${run_result} >> ${run_benchmark_result_file}

  223.             return 1

  224.         fi

  225.         # run benchmark test without clib data

  226.         echo ${model_name} >> "${run_benchmark_log_file}"

  227.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  228.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "{run_benchmark_log_file}"

  229.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  230.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  231.         if [ $? = 0 ]; then

  232.             run_result='arm64: '${model_name}' pass'

  233.             echo ${run_result} >> ${run_benchmark_result_file}

  234.         else

  235.             run_result='arm64: '${model_name}' failed'

  236.             echo ${run_result} >> ${run_benchmark_result_file}

  237.             return 1

  238.         fi

  239. 	#sleep 1

  240.     done < ${models_onnx_config}

  241. 

  242.     # Run fp16 converted models:

  243.     while read line; do

  244.         model_name=${line}

  245.         if [[ $model_name == \#* ]]; then

  246.           continue

  247.         fi

  248.         echo ${model_name} >> "${run_benchmark_log_file}"

  249.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  250.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --fp16Priority=true --accuracyThreshold=5' >> "${run_benchmark_log_file}"

  251.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --fp16Priority=true --accuracyThreshold=5' >> adb_run_cmd.txt

  252.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  253.         if [ $? = 0 ]; then

  254.             run_result='arm64: '${model_name}' pass'

  255.             echo ${run_result} >> ${run_benchmark_result_file}

  256.         else

  257.             run_result='arm64: '${model_name}' failed'

  258.             echo ${run_result} >> ${run_benchmark_result_file}

  259.             return 1

  260.         fi

  261.         # run benchmark test without clib data

  262.         echo ${model_name} >> "${run_benchmark_log_file}"

  263.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  264.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2 --fp16Priority=true --accuracyThreshold=5' >> "${run_benchmark_log_file}"

  265.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2 --fp16Priority=true --accuracyThreshold=5' >> adb_run_cmd.txt

  266.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  267.         if [ $? = 0 ]; then

  268.             run_result='arm64: '${model_name}' pass'

  269.             echo ${run_result} >> ${run_benchmark_result_file}

  270.         else

  271.             run_result='arm64: '${model_name}' failed'

  272. 	    echo ${run_result} >> ${run_benchmark_result_file}

  273.             return 1

  274.         fi

  275. 	#sleep 1

  276.     done < ${models_fp16_config}

  277. 

  278.     # Run tflite aware training quantization converted models:

  279.     while read line; do

  280.         model_name=${line}

  281.         if [[ $model_name == \#* ]]; then

  282.           continue

  283.         fi

  284.         echo ${model_name} >> "${run_benchmark_log_file}"

  285.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  286.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --fp16Priority=true --accuracyThreshold=5' >> "${run_benchmark_log_file}"

  287.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1 --fp16Priority=true --accuracyThreshold=5' >> adb_run_cmd.txt

  288.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  289.         if [ $? = 0 ]; then

  290.             run_result='arm64: '${model_name}' pass'

  291.             echo ${run_result} >> ${run_benchmark_result_file}

  292.         else

  293.             run_result='arm64: '${model_name}' failed'

  294.             echo ${run_result} >> ${run_benchmark_result_file}

  295.             return 1

  296.         fi

  297.         # run benchmark test without clib data

  298.         echo ${model_name} >> "${run_benchmark_log_file}"

  299.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  300.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2 --fp16Priority=true --accuracyThreshold=5' >> "${run_benchmark_log_file}"

  301.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2 --fp16Priority=true --accuracyThreshold=5' >> adb_run_cmd.txt

  302.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  303.         if [ $? = 0 ]; then

  304.             run_result='arm64: '${model_name}' pass'

  305.             echo ${run_result} >> ${run_benchmark_result_file}

  306.         else

  307.             run_result='arm64: '${model_name}' failed'

  308.             echo ${run_result} >> ${run_benchmark_result_file}

  309.             return 1

  310.         fi

  311. 	#sleep 1

  312.     done < ${models_tflite_awaretraining_config}

  313. 

  314.     # Run gpu tflite converted models:

  315.     while read line; do

  316.         model_name=${line}

  317.         if [[ $model_name == \#* ]]; then

  318.           continue

  319.         fi

  320.         echo ${model_name} >> "${run_benchmark_log_file}"

  321.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  322.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> "${run_benchmark_log_file}"

  323.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --modelPath='${model_name}'.ms --inDataPath=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --calibDataPath=/data/local/tmp/input_output/output/'${model_name}'.ms.out --warmUpLoopCount=1 --loopCount=1' >> adb_run_cmd.txt

  324.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  325.         if [ $? = 0 ]; then

  326.             run_result='arm64_gpu: '${model_name}' pass'

  327.             echo ${run_result} >> ${run_benchmark_result_file}

  328.         else

  329.             run_result='arm64_gpu: '${model_name}' failed'

  330.             echo ${run_result} >> ${run_benchmark_result_file}

  331.             return 1

  332.         fi

  333.         # run benchmark test without clib data

  334.         #echo ${model_name}

  335.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  336.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_benchmark_log_file}"

  337.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --modelPath='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  338.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_benchmark_log_file}"

  339.         if [ $? = 0 ]; then

  340.             run_result='arm64_gpu: '${model_name}' pass'

  341.             echo ${run_result} >> ${run_benchmark_result_file}

  342.         else

  343.             run_result='arm64_gpu: '${model_name}' failed'

  344.             echo ${run_result} >> ${run_benchmark_result_file}

  345.             return 1

  346.         fi

  347. 	#sleep 1

  348.     done < ${models_tflite_gpu_config}

  349. }

  350. 

  351. # Print start msg before run testcase

  352. function MS_PRINT_TESTCASE_START_MSG() {

  353.     echo ""

  354.     echo -e "-----------------------------------------------------------------------------------------------------------------------------------"

  355.     echo -e "env        Testcase                                                                                                       Result   "

  356.     echo -e "---        --------                                                                                                       ------   "

  357. }

  358. 

  359. # Print start msg after run testcase

  360. function MS_PRINT_TESTCASE_END_MSG() {

  361.     echo -e "-----------------------------------------------------------------------------------------------------------------------------------"

  362. }

  363. 

  364. 

  365. basepath=$(pwd)

  366. echo ${basepath}

  367. #set -e

  368. 

  369. # Example:sh run_benchmark_nets.sh -a /home/temp_test -c /home/temp_test -m /home/temp_test/models -d "8KE5T19620002408"

  370. while getopts "a:c:m:d:" opt; do

  371.     case ${opt} in

  372.         a)

  373. 	    arm_path=${OPTARG}

  374.             echo "arm_path is ${OPTARG}"

  375.             ;;

  376.         c)

  377. 	    convertor_path=${OPTARG}

  378.             echo "convertor_path is ${OPTARG}"

  379.             ;;

  380.         m)

  381. 	    models_path=${OPTARG}

  382.             echo "models_path is ${OPTARG}"

  383.             ;;

  384.         d)

  385. 	    device_id=${OPTARG}

  386.             echo "device_id is ${OPTARG}"

  387.             ;;

  388.         ?)

  389.         echo "unknown para"

  390.         exit 1;;

  391.     esac

  392. done

  393. 

  394. mkdir train

  395. mv ${arm_path}/*runtime-*train* ./train 

  396. file_name=$(ls ${arm_path}/*runtime-arm64*.tar.gz)

  397. IFS="-" read -r -a file_name_array <<< "$file_name"

  398. version=${file_name_array[2]}

  399. IFS="." read -r -a suffix <<< "${file_name_array[-1]}"

  400. process_unit_arm=${suffix[0]}

  401. 

  402. file_name=$(ls ${convertor_path}/*runtime-x86*.tar.gz)

  403. IFS="-" read -r -a file_name_array <<< "$file_name"

  404. IFS="." read -r -a suffix <<< "${file_name_array[-1]}"

  405. process_unit_x86=${suffix[0]}

  406. 

  407. # Unzip arm

  408. cd ${arm_path} || exit 1

  409. tar -zxf mindspore-lite-${version}-runtime-arm64-${process_unit_arm}.tar.gz || exit 1

  410. 

  411. # Unzip x86 runtime and convertor

  412. cd ${convertor_path} || exit 1

  413. tar -zxf mindspore-lite-${version}-runtime-x86-${process_unit_x86}.tar.gz || exit 1

  414. 

  415. tar -zxf mindspore-lite-${version}-converter-ubuntu.tar.gz || exit 1

  416. cd ${convertor_path}/mindspore-lite-${version}-converter-ubuntu || exit 1

  417. cp converter/converter_lite ./ || exit 1

  418. export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./lib/:./third_party/protobuf/lib:./third_party/flatbuffers/lib

  419. 

  420. # Convert the models

  421. cd ${convertor_path}/mindspore-lite-${version}-converter-ubuntu || exit 1

  422. 

  423. # Write resulte to temp file

  424. run_benchmark_result_file=${basepath}/run_benchmark_result.txt

  425. echo ' ' > ${run_benchmark_result_file}

  426. 

  427. run_benchmark_log_file=${basepath}/run_benchmark_log.txt

  428. echo 'run benchmark logs: ' > ${run_benchmark_log_file}

  429. 

  430. # Set models config filepath

  431. models_tflite_config=${basepath}/models_tflite.cfg

  432. models_caffe_config=${basepath}/models_caffe.cfg

  433. models_tflite_awaretraining_config=${basepath}/models_tflite_awaretraining.cfg

  434. models_tflite_posttraining_config=${basepath}/models_tflite_posttraining.cfg

  435. models_onnx_config=${basepath}/models_onnx.cfg

  436. models_fp16_config=${basepath}/models_fp16.cfg

  437. models_mindspore_config=${basepath}/models_mindspore.cfg

  438. models_tflite_gpu_config=${basepath}/models_tflite_gpu.cfg

  439. Convert_status=0

  440. 

  441. rm -rf ${basepath}/ms_models

  442. mkdir -p ${basepath}/ms_models

  443. ms_models_path=${basepath}/ms_models

  444. 

  445. # Convert tflite models:

  446. while read line; do

  447.     model_name=${line}

  448.     if [[ $model_name == \#* ]]; then

  449.       continue

  450.     fi

  451.     echo ${model_name} >> "${run_benchmark_log_file}"

  452.     echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_benchmark_log_file}"

  453.     ./converter_lite  --fmk=TFLITE --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name} || Convert_status=$?

  454. done < ${models_tflite_config}

  455. 

  456. # Convert caffe models:

  457. while read line; do

  458.     model_name=${line}

  459.     if [[ $model_name == \#* ]]; then

  460.       continue

  461.     fi

  462.     echo ${model_name} >> "${run_benchmark_log_file}"

  463.     #pwd >> ${run_benchmark_log_file}

  464.     echo './converter_lite  --fmk=CAFFE --modelFile='${models_path}'/'${model_name}'.prototxt --weightFile='${models_path}'/'${model_name}'.caffemodel --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_benchmark_log_file}"

  465.     ./converter_lite  --fmk=CAFFE --modelFile=${models_path}/${model_name}.prototxt --weightFile=${models_path}/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name} || Convert_status=$?

  466. done < ${models_caffe_config}

  467. 

  468. # Convert onnx models:

  469. while read line; do

  470.     model_name=${line}

  471.     if [[ $model_name == \#* ]]; then

  472.       continue

  473.     fi

  474.     echo ${model_name} >> "${run_benchmark_log_file}"

  475.     #pwd >> ${run_benchmark_log_file}

  476.     echo './converter_lite  --fmk=ONNX --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_benchmark_log_file}"

  477.     ./converter_lite  --fmk=ONNX --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name} || Convert_status=$?

  478. done < ${models_onnx_config}

  479. 

  480. # Convert mindspore models:

  481. while read line; do

  482.     model_name=${line}

  483.     if [[ $model_name == \#* ]]; then

  484.       continue

  485.     fi

  486.     echo ${model_name} >> "${run_benchmark_log_file}"

  487.     pwd >> "${run_benchmark_log_file}"

  488.     echo './converter_lite  --fmk=MS --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_benchmark_log_file}"

  489.     ./converter_lite  --fmk=MS --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name} || Convert_status=$?

  490. done < ${models_mindspore_config}

  491. 

  492. # Convert TFLite PostTraining models:

  493. while read line; do

  494.     model_name=${line}

  495.     if [[ $model_name == \#* ]]; then

  496.       continue

  497.     fi

  498.     echo ${model_name} >> "${run_benchmark_log_file}"

  499.     echo 'convert mode name: '${model_name}' begin.'

  500.     echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}_posttraining' --quantType=PostTraining --config_file='${models_path}'/'${model_name}'_posttraining.config' >> "${run_benchmark_log_file}"

  501.     ./converter_lite  --fmk=TFLITE --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}_posttraining --quantType=PostTraining --config_file=${models_path}/${model_name}_posttraining.config || Convert_status=$?

  502. done < ${models_tflite_posttraining_config}

  503.     

  504. # Convert TFLite AwareTraining models:

  505. while read line; do

  506.     model_name=${line}

  507.     if [[ $model_name == \#* ]]; then

  508.       continue

  509.     fi

  510.     echo ${model_name} >> "${run_benchmark_log_file}"

  511.     echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}' --quantType=AwareTraining' >> "${run_benchmark_log_file}"

  512.     ./converter_lite  --fmk=TFLITE --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name} --quantType=AwareTraining || Convert_status=$?

  513. done < ${models_tflite_awaretraining_config}

  514. 

  515. # Copy fp16 ms models:

  516. while read line; do

  517.   model_name=${line%.*}

  518.   if [[ $model_name == \#* ]]; then

  519.       continue

  520.   fi

  521.   echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'

  522.   cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms

  523. done < ${models_fp16_config}

  524. 

  525. # Check all result and return value

  526. if [[ ${Convert_status} = 0 ]];then

  527.     echo "convert is ended"

  528. else

  529.     echo "convert failed"

  530.     cat ${run_benchmark_log_file}

  531.     exit 1

  532. fi

  533. 

  534. # Push to the arm and run benchmark:

  535. # First:copy benchmark exe and so files to the server which connected to the phone

  536. echo "Push files to the arm and run benchmark"

  537. rm -rf ${basepath}/benchmark_test

  538. mkdir -p ${basepath}/benchmark_test

  539. benchmark_test_path=${basepath}/benchmark_test

  540. cd ${benchmark_test_path} || exit 1

  541. 

  542. # If build with minddata, copy the minddata related libs

  543. if [ -f ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/libminddata-lite.so ]; then

  544.     cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/libjpeg-turbo/lib/libjpeg.so ${benchmark_test_path}/libjpeg.so || exit 1

  545.     cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/libjpeg-turbo/lib/libturbojpeg.so ${benchmark_test_path}/libturbojpeg.so || exit 1

  546.     cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/opencv/lib/libopencv_core.so ${benchmark_test_path}/libopencv_core.so || exit 1

  547.     cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/opencv/lib/libopencv_imgcodecs.so ${benchmark_test_path}/libopencv_imgcodecs.so || exit 1

  548.     cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/third_party/opencv/lib/libopencv_imgproc.so ${benchmark_test_path}/libopencv_imgproc.so || exit 1

  549.     cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  550. fi

  551. 

  552. cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  553. cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/lib/liboptimize.so ${benchmark_test_path}/liboptimize.so || exit 1

  554. cp -a ${arm_path}/mindspore-lite-${version}-runtime-arm64-${process_unit_arm}/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  555. 

  556. # Copy the MindSpore models:

  557. cp -a ${ms_models_path}/*.ms ${benchmark_test_path} || exit 1

  558. 

  559. # Second:adb push all needed files to the phone

  560. adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  561. 

  562. # Third:run adb ,run session ,check the result:

  563. echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  564. echo 'cp  /data/local/tmp/libc++_shared.so ./' >> adb_cmd.txt

  565. echo 'chmod 777 benchmark' >> adb_cmd.txt

  566. 

  567. adb -s ${device_id} shell < adb_cmd.txt

  568. 

  569. # Run on x86

  570. echo "start Run x86 ..."

  571. Run_x86 &

  572. Run_x86_PID=$!

  573. sleep 1

  574. 

  575. # Run on arm64

  576. echo "start Run arm64 ..."

  577. Run_arm64 &

  578. Run_arm64_PID=$!

  579. 

  580. wait ${Run_x86_PID}

  581. Run_x86_status=$?

  582. 

  583. wait ${Run_arm64_PID}

  584. Run_arm64_status=$?

  585. 

  586. # Print all results:

  587. MS_PRINT_TESTCASE_START_MSG

  588. while read line; do

  589.     arr=("${line}")

  590.     printf "%-10s %-110s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]}

  591. done < ${run_benchmark_result_file}

  592. MS_PRINT_TESTCASE_END_MSG

  593. 

  594. # Check all result and return value

  595. if [[ ${Run_x86_status} = 0 ]] && [[ ${Run_arm64_status} = 0 ]];then

  596.     echo "Run_x86 and Run_arm64 is ended"

  597.     exit 0

  598. else

  599.     echo "run failed"

  600.     cat ${run_benchmark_log_file}

  601.     

  602.     #print the result table again:

  603.     MS_PRINT_TESTCASE_START_MSG

  604.     while read line; do

  605.         arr=("${line}")

  606.         printf "%-10s %-110s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]}

  607.     done < ${run_benchmark_result_file}

  608.     MS_PRINT_TESTCASE_END_MSG

  609.     

  610.     exit 1

  611. fi