zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 6fcd6cab68
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '6fcd6cab68'
${ noResults }
mindspore2022/mindspore/lite/test/run_benchmark_nets.sh

3030 lines
192 kB
Raw Blame History 

  1. #!/bin/bash

  2. 

  3. # Run converter on x86 platform:

  4. function Run_Converter() {

  5.     # Unzip x86 runtime and converter

  6.     cd ${x86_path} || exit 1

  7.     tar -zxf mindspore-lite-${version}-inference-linux-x64.tar.gz || exit 1

  8.     cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64/ || exit 1

  9. 

  10.     cp tools/converter/converter/converter_lite ./ || exit 1

  11.     export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./tools/converter/lib/:./tools/converter/third_party/glog/lib

  12. 

  13.     rm -rf ${ms_models_path}

  14.     mkdir -p ${ms_models_path}

  15. 

  16.     # Convert tf models:

  17.     while read line; do

  18.         model_name=${line%;*}

  19.         if [[ $model_name == \#* ]]; then

  20.           continue

  21.         fi

  22.         echo ${model_name} >> "${run_converter_log_file}"

  23.         echo './converter_lite  --fmk=TF --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_converter_log_file}"

  24.         ./converter_lite  --fmk=TF --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}

  25.         if [ $? = 0 ]; then

  26.             converter_result='converter tf '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  27.         else

  28.             converter_result='converter tf '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  29.         fi

  30.     done < ${models_tf_config}

  31. 

  32.     # Convert tflite models:

  33.     while read line; do

  34.         model_name=${line}

  35.         if [[ $model_name == \#* ]]; then

  36.           continue

  37.         fi

  38.         echo ${model_name} >> "${run_converter_log_file}"

  39.         echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_converter_log_file}"

  40.         ./converter_lite  --fmk=TFLITE --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}

  41.         if [ $? = 0 ]; then

  42.             converter_result='converter tflite '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  43.         else

  44.             converter_result='converter tflite '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  45.         fi

  46.     done < ${models_tflite_config}

  47. 

  48.     # Convert caffe models:

  49.     while read line; do

  50.         model_name=${line}

  51.         if [[ $model_name == \#* ]]; then

  52.           continue

  53.         fi

  54.         echo ${model_name} >> "${run_converter_log_file}"

  55.         echo './converter_lite  --fmk=CAFFE --modelFile='${models_path}'/'${model_name}'.prototxt --weightFile='${models_path}'/'${model_name}'.caffemodel --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_converter_log_file}"

  56.         ./converter_lite  --fmk=CAFFE --modelFile=${models_path}/${model_name}.prototxt --weightFile=${models_path}/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name}

  57.         if [ $? = 0 ]; then

  58.             converter_result='converter caffe '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  59.         else

  60.             converter_result='converter caffe '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  61.         fi

  62.     done < ${models_caffe_config}

  63. 

  64.     # Convert onnx models:

  65.     while read line; do

  66.         model_name=${line%;*}

  67.         if [[ $model_name == \#* ]]; then

  68.           continue

  69.         fi

  70.         echo ${model_name} >> "${run_converter_log_file}"

  71.         echo './converter_lite  --fmk=ONNX --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_converter_log_file}"

  72.         ./converter_lite  --fmk=ONNX --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}

  73.         if [ $? = 0 ]; then

  74.             converter_result='converter onnx '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  75.         else

  76.             converter_result='converter onnx '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  77.         fi

  78.     done < ${models_onnx_config}

  79. 

  80.     # Convert mindspore models:

  81.     while read line; do

  82.         mindspore_line_info=${line}

  83.         if [[ $mindspore_line_info == \#* ]]; then

  84.           continue

  85.         fi

  86.         model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'`

  87.         accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'`

  88.         echo ${model_name} >> "${run_converter_log_file}"

  89.         echo './converter_lite  --fmk=MINDIR --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_converter_log_file}"

  90.         ./converter_lite  --fmk=MINDIR --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}

  91.         if [ $? = 0 ]; then

  92.             converter_result='converter mindspore '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  93.         else

  94.             converter_result='converter mindspore '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  95.         fi

  96.     done < ${models_mindspore_config}

  97. 

  98.     # Convert mindspore train models:

  99.     while read line; do

  100.         model_name=${line}

  101.         if [[ $model_name == \#* ]]; then

  102.           continue

  103.         fi

  104.         echo ${model_name}'_train' >> "${run_converter_log_file}"

  105.         echo './converter_lite  --fmk=MINDIR --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'_train  --trainModel=true' >> "${run_converter_log_file}"

  106.         ./converter_lite  --fmk=MINDIR --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}'_train' --trainModel=true

  107.         if [ $? = 0 ]; then

  108.             converter_result='converter mindspore '${model_name}'_train pass';echo ${converter_result} >> ${run_converter_result_file}

  109.         else

  110.             converter_result='converter mindspore '${model_name}'_train failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  111.         fi

  112.     done < ${models_mindspore_train_config}

  113. 

  114.     # Convert TFLite PostTraining models:

  115.     while read line; do

  116.         posttraining_line_info=${line}

  117.         if [[ $posttraining_line_info == \#* ]]; then

  118.           continue

  119.         fi

  120.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  121.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  122.         echo ${model_name} >> "${run_converter_log_file}"

  123.         echo 'convert mode name: '${model_name}' begin.'

  124.         echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}_posttraining' --quantType=PostTraining --config_file='${models_path}'/'${model_name}'_posttraining.config' >> "${run_converter_log_file}"

  125.         ./converter_lite  --fmk=TFLITE --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}_posttraining --quantType=PostTraining --configFile=${models_path}/${model_name}_posttraining.config

  126.         if [ $? = 0 ]; then

  127.             converter_result='converter post_training '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  128.         else

  129.             converter_result='converter post_training '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  130.         fi

  131.     done < ${models_tflite_posttraining_config}

  132. 

  133.     # Convert Caffe PostTraining models:

  134.     while read line; do

  135.         posttraining_line_info=${line}

  136.         if [[ $posttraining_line_info == \#* ]]; then

  137.           continue

  138.         fi

  139.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  140.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  141.         echo ${model_name} >> "${run_converter_log_file}"

  142.         echo 'convert mode name: '${model_name}' begin.'

  143.         echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}_posttraining' --quantType=PostTraining --config_file='${models_path}'/'${model_name}'_posttraining.config' >> "${run_converter_log_file}"

  144.         ./converter_lite  --fmk=CAFFE --modelFile=$models_path/${model_name}.prototxt --weightFile=$models_path/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name}_posttraining --quantType=PostTraining --configFile=${models_path}/config.${model_name}

  145.         if [ $? = 0 ]; then

  146.             converter_result='converter post_training '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  147.         else

  148.             converter_result='converter post_training '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  149.         fi

  150.     done < ${models_caffe_posttraining_config}

  151. 

  152.     # Convert TFLite AwareTraining models:

  153.     while read line; do

  154.         model_name=${line}

  155.         if [[ $model_name == \#* ]]; then

  156.           continue

  157.         fi

  158.         echo ${model_name} >> "${run_converter_log_file}"

  159.         echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}' --inputDataType=FLOAT  --outputDataType=FLOAT' >> "${run_converter_log_file}"

  160.         ./converter_lite  --fmk=TFLITE --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}   --inputDataType=FLOAT  --outputDataType=FLOAT

  161.         if [ $? = 0 ]; then

  162.             converter_result='converter aware_training '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  163.         else

  164.             converter_result='converter aware_training '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  165.         fi

  166.     done < ${models_tflite_awaretraining_config}

  167. 

  168.     # Convert tflite weightquant models:

  169.     while read line; do

  170.         weight_quant_line_info=${line}

  171.         if [[ $weight_quant_line_info == \#* ]]; then

  172.           continue

  173.         fi

  174.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  175.         echo ${model_name} >> "${run_converter_log_file}"

  176.         echo './converter_lite  --fmk=TFLITE --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}_weightquant'--quantType=WeightQuant --bitNum=8 --quantWeightChannel=0' >> "${run_converter_log_file}"

  177.         ./converter_lite  --fmk=TFLITE --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}_weightquant --quantType=WeightQuant --bitNum=8 --quantWeightChannel=0

  178.         if [ $? = 0 ]; then

  179.             converter_result='converter weight_quant '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  180.         else

  181.             converter_result='converter weight_quant '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  182.         fi

  183.     done < ${models_tflite_weightquant_config}

  184. 

  185.     # Convert caffe weightquant models:

  186.     while read line; do

  187.         weight_quant_line_info=${line}

  188.         if [[ ${weight_quant_line_info} == \#* ]]; then

  189.           continue

  190.         fi

  191.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  192.         echo ${model_name} >> "${run_converter_log_file}"

  193.         echo './converter_lite  --fmk=CAFFE --modelFile='${models_path}'/'${model_name}'.prototxt --weightFile='${models_path}'/'${model_name}'.caffemodel --outputFile='${ms_models_path}'/'${model_name}_weightquant'  --quantType=WeightQuant --bitNum=8 --quantWeightChannel=0' >> "${run_converter_log_file}"

  194.         ./converter_lite  --fmk=CAFFE --modelFile=${models_path}/${model_name}.prototxt --weightFile=${models_path}/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name}_weightquant  --quantType=WeightQuant --bitNum=8 --quantWeightChannel=0

  195.         if [ $? = 0 ]; then

  196.             converter_result='converter caffe_weight_quant '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  197.         else

  198.             converter_result='converter caffe_weight_quant '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  199.         fi

  200.     done < ${models_caffe_weightquant_config}

  201. 

  202.     # Convert mindir weightquant models:

  203.     while read line; do

  204.         weight_quant_line_info=${line}

  205.         if [[ $weight_quant_line_info == \#* ]]; then

  206.           continue

  207.         fi

  208.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  209.         echo ${model_name} >> "${run_converter_log_file}"

  210.         echo './converter_lite  --fmk=MINDIR --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}' --quantType=WeightQuant --bitNum=8 --quantWeightSize=0 --quantWeightChannel=0' >> "${run_converter_log_file}"

  211.         ./converter_lite  --fmk=MINDIR --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}_weightquant --quantType=WeightQuant --bitNum=8 --quantWeightSize=0 --quantWeightChannel=0

  212.         if [ $? = 0 ]; then

  213.             converter_result='converter weight_quant '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  214.         else

  215.             converter_result='converter weight_quant '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  216.         fi

  217.     done < ${models_mindspore_weightquant_config}

  218. 

  219.     # Convert tf weightquant models:

  220.     while read line; do

  221.         weight_quant_line_info=${line}

  222.         if [[ $weight_quant_line_info == \#* ]]; then

  223.           continue

  224.         fi

  225.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  226.         echo ${model_name} >> "${run_converter_log_file}"

  227.         echo './converter_lite  --fmk=TF --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'_weightquant' >> "${run_converter_log_file}"

  228.         ./converter_lite  --fmk=TF --modelFile=$models_path/${model_name} --outputFile=${ms_models_path}/${model_name}_weightquant --quantType=WeightQuant --bitNum=8 --quantWeightChannel=0

  229.         if [ $? = 0 ]; then

  230.             converter_result='converter weight_quant '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  231.         else

  232.             converter_result='converter weight_quant '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  233.         fi

  234.     done < ${models_tf_weightquant_config}

  235. 

  236.     # Convert mindir mixbit weightquant models:

  237.     while read line; do

  238.         line_info=${line}

  239.         if [[ $line_info == \#* ]]; then

  240.           continue

  241.         fi

  242.         model_name=`echo ${line_info}|awk -F ' ' '{print $1}'`

  243. 

  244.         echo ${model_name}'_7bit' >> "${run_converter_log_file}"

  245.         echo './converter_lite  --fmk=MINDIR --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'_7bit  --quantType=WeightQuant --bitNum=7 --quantWeightSize=0 --quantWeightChannel=0' >> "${run_converter_log_file}"

  246.         ./converter_lite  --fmk=MINDIR --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}'_7bit' --quantType=WeightQuant --bitNum=7 --quantWeightSize=0 --quantWeightChannel=0

  247.         if [ $? = 0 ]; then

  248.             converter_result='converter mindspore '${model_name}'_7bit pass';echo ${converter_result} >> ${run_converter_result_file}

  249.         else

  250.             converter_result='converter mindspore '${model_name}'_7bit failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  251.         fi

  252.         echo ${model_name}'_9bit' >> "${run_converter_log_file}"

  253.         echo './converter_lite  --fmk=MINDIR --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'_9bit  --quantType=WeightQuant --bitNum=9 --quantWeightSize=0 --quantWeightChannel=0' >> "${run_converter_log_file}"

  254.         ./converter_lite  --fmk=MINDIR --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}'_9bit' --quantType=WeightQuant --bitNum=9 --quantWeightSize=0 --quantWeightChannel=0

  255.         if [ $? = 0 ]; then

  256.             converter_result='converter mindspore '${model_name}'_9bit pass';echo ${converter_result} >> ${run_converter_result_file}

  257.         else

  258.             converter_result='converter mindspore '${model_name}'_9bit failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  259.         fi

  260.     done < ${models_mindspore_mixbit_config}

  261. 

  262.     # Convert models which has multiple inputs:

  263.     while read line; do

  264.         if [[ $line == \#* ]]; then

  265.           continue

  266.         fi

  267.         model_name=${line%%;*}

  268.         model_type=${model_name##*.}

  269.         case $model_type in

  270.           pb)

  271.             model_fmk="TF"

  272.             ;;

  273.           tflite)

  274.             model_fmk="TFLITE"

  275.             ;;

  276.           onnx)

  277.             model_fmk="ONNX"

  278.             ;;

  279.           mindir)

  280.             model_fmk="MINDIR"

  281.             ;;

  282.           *)

  283.             model_type="caffe"

  284.             model_fmk="CAFFE"

  285.             ;;

  286.         esac

  287.         if [[ $model_fmk == "CAFFE" ]]; then

  288.           echo ${model_name} >> "${run_converter_log_file}"

  289.           echo './converter_lite  --fmk='${model_fmk}' --modelFile='$models_path/${model_name}'.prototxt --weightFile='$models_path'/'${model_name}'.caffemodel --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}"

  290.           ./converter_lite  --fmk=${model_fmk} --modelFile=${models_path}/${model_name}.prototxt --weightFile=${models_path}/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name}

  291.         else

  292.           echo ${model_name} >> "${run_converter_log_file}"

  293.           echo './converter_lite  --fmk='${model_fmk}' --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}"

  294.           ./converter_lite  --fmk=${model_fmk} --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}

  295.         fi

  296.         if [ $? = 0 ]; then

  297.             converter_result='converter '${model_type}' '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  298.         else

  299.             converter_result='converter '${model_type}' '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  300.         fi

  301.     done < ${models_with_multiple_inputs_config}

  302. 

  303.     # Convert models which does not need to be cared about the accuracy:

  304.     while read line; do

  305.         if [[ $line == \#* ]]; then

  306.           continue

  307.         fi

  308.         model_name=${line%%;*}

  309.         model_type=${model_name##*.}

  310.         case $model_type in

  311.           pb)

  312.             model_fmk="TF"

  313.             ;;

  314.           tflite)

  315.             model_fmk="TFLITE"

  316.             ;;

  317.           onnx)

  318.             model_fmk="ONNX"

  319.             ;;

  320.           mindir)

  321.             model_fmk="MINDIR"

  322.             ;;

  323.           *)

  324.             model_type="caffe"

  325.             model_fmk="CAFFE"

  326.             ;;

  327.         esac

  328.         if [[ $model_fmk == "CAFFE" ]]; then

  329.           echo ${model_name} >> "${run_converter_log_file}"

  330.           echo './converter_lite  --fmk='${model_fmk}' --modelFile='$models_path/${model_name}'.prototxt --weightFile='$models_path'/'${model_name}'.caffemodel --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}"

  331.           ./converter_lite  --fmk=${model_fmk} --modelFile=${models_path}/${model_name}.prototxt --weightFile=${models_path}/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name}

  332.         else

  333.           echo ${model_name} >> "${run_converter_log_file}"

  334.           echo './converter_lite  --fmk='${model_fmk}' --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}"

  335.           ./converter_lite  --fmk=${model_fmk} --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}

  336.         fi

  337.         if [ $? = 0 ]; then

  338.             converter_result='converter '${model_type}' '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  339.         else

  340.             converter_result='converter '${model_type}' '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  341.         fi

  342.     done < ${models_for_process_only_config}

  343. 

  344.     # Copy fp16 ms models:

  345.     while read line; do

  346.         fp16_line_info=${line}

  347.         if [[ $fp16_line_info == \#* ]]; then

  348.           continue

  349.         fi

  350.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  351.         model_name=${model_info%%;*}

  352.         echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'

  353.         cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms

  354.         if [ $? = 0 ]; then

  355.             converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  356.         else

  357.             converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  358.         fi

  359.     done < ${models_onnx_fp16_config}

  360. 

  361.     while read line; do

  362.         fp16_line_info=${line}

  363.         if [[ $fp16_line_info == \#* ]]; then

  364.           continue

  365.         fi

  366.         model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  367.         echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'

  368.         cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms

  369.         if [ $? = 0 ]; then

  370.             converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  371.         else

  372.             converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  373.         fi

  374.     done < ${models_caffe_fp16_config}

  375. 

  376.     while read line; do

  377.         fp16_line_info=${line}

  378.         if [[ $fp16_line_info == \#* ]]; then

  379.           continue

  380.         fi

  381.         model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  382.         echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'

  383.         cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms

  384.         if [ $? = 0 ]; then

  385.             converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  386.         else

  387.             converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  388.         fi

  389.     done < ${models_tflite_fp16_config}

  390. 

  391.     while read line; do

  392.         fp16_line_info=${line}

  393.         if [[ $fp16_line_info == \#* ]]; then

  394.           continue

  395.         fi

  396.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  397.         model_name=${model_info%%;*}

  398.         echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'

  399.         cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms

  400.         if [ $? = 0 ]; then

  401.             converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  402.         else

  403.             converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  404.         fi

  405.     done < ${models_tf_fp16_config}

  406. 

  407.     while read line; do

  408.       fp16_line_info=${line}

  409.       if [[ $fp16_line_info == \#* ]]; then

  410.         continue

  411.       fi

  412.       model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  413.       model_name=${model_info%%;*}

  414.       echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'

  415.       cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms

  416.       if [ $? = 0 ]; then

  417.           converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}

  418.       else

  419.           converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1

  420.       fi

  421.     done < ${models_multiple_inputs_fp16_config}

  422. }

  423. 

  424. # Run on x86 platform:

  425. function Run_x86() {

  426.     echo 'cd  '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64' >> "${run_x86_log_file}"

  427.     cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64 || return 1

  428.     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./inference/lib

  429.     cp tools/benchmark/benchmark ./ || exit 1

  430. 

  431.     # Run tf converted models:

  432.     while read line; do

  433.         model_name=${line%;*}

  434.         length=${#model_name}

  435.         input_shapes=${line:length+1}

  436.         if [[ $model_name == \#* ]]; then

  437.           continue

  438.         fi

  439.         echo ${model_name} >> "${run_x86_log_file}"

  440.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  441.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --inputShapes=${input_shapes} --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_log_file}"

  442.         if [ $? = 0 ]; then

  443.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  444.         else

  445.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  446.         fi

  447.     done < ${models_tf_config}

  448. 

  449.     # Run tflite converted models:

  450.     while read line; do

  451.         model_name=${line}

  452.         if [[ $model_name == \#* ]]; then

  453.           continue

  454.         fi

  455.         echo ${model_name} >> "${run_x86_log_file}"

  456.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  457.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_log_file}"

  458.         if [ $? = 0 ]; then

  459.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  460.         else

  461.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  462.         fi

  463.     done < ${models_tflite_config}

  464. 

  465.     # Run caffe converted models:

  466.     while read line; do

  467.         model_name=${line}

  468.         if [[ $model_name == \#* ]]; then

  469.           continue

  470.         fi

  471.         echo ${model_name} >> "${run_x86_log_file}"

  472.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  473.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_log_file}"

  474.         if [ $? = 0 ]; then

  475.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  476.         else

  477.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  478.         fi

  479.     done < ${models_caffe_config}

  480. 

  481.     # Run onnx converted models:

  482.     while read line; do

  483.         model_name=${line%;*}

  484.         length=${#model_name}

  485.         input_shapes=${line:length+1}

  486.         if [[ $model_name == \#* ]]; then

  487.           continue

  488.         fi

  489.         echo ${model_name} >> "${run_x86_log_file}"

  490.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  491.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --inputShapes=${input_shapes} --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_log_file}"

  492.         if [ $? = 0 ]; then

  493.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  494.         else

  495.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  496.         fi

  497.     done < ${models_onnx_config}

  498. 

  499.     # Run mindspore converted models:

  500.     while read line; do

  501.         mindspore_line_info=${line}

  502.         if [[ $mindspore_line_info == \#* ]]; then

  503.           continue

  504.         fi

  505.         model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'`

  506.         accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'`

  507.         echo "---------------------------------------------------------" >> "${run_x86_log_file}"

  508.         echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_x86_log_file}"

  509.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  510.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  511.         if [ $? = 0 ]; then

  512.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  513.         else

  514.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  515.         fi

  516.     done < ${models_mindspore_config}

  517. 

  518.     # Run mindspore converted train models:

  519.     while read line; do

  520.         model_name=${line}

  521.         if [[ $model_name == \#* ]]; then

  522.           continue

  523.         fi

  524.         echo ${model_name}'_train' >> "${run_x86_log_file}"

  525.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_train.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=${models_path}/input_output/output/'${model_name}'.train.ms.out' >> "${run_x86_log_file}"

  526.         ./benchmark --modelFile=${ms_models_path}/${model_name}'_train'.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.train.ms.out --accuracyThreshold=1.5 >> "${run_x86_log_file}"

  527.         if [ $? = 0 ]; then

  528.             run_result='x86: '${model_name}'_train pass'; echo ${run_result} >> ${run_benchmark_result_file}

  529.         else

  530.             run_result='x86: '${model_name}'_train failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  531.         fi

  532.     done < ${models_mindspore_train_config}

  533. 

  534.     # Run tflite post training quantization converted models:

  535.     while read line; do

  536.         posttraining_line_info=${line}

  537.         if [[ $posttraining_line_info == \#* ]]; then

  538.           continue

  539.         fi

  540.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  541.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  542.         transformer_data_path="${models_path}/input_output/input"

  543.         echo ${model_name} >> "${run_x86_log_file}"

  544.         if [[ $model_name == "mobilenet.tflite" ]]; then

  545.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  546.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  547.         fi

  548.         if [[ $model_name == "transformer_20200831_encoder_fp32.tflite" ]]; then

  549.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_0-35.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  550.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_35.bin,${transformer_data_path}/encoder_buffer_in_0.bin,${transformer_data_path}/encoder_buffer_in_1.bin,${transformer_data_path}/encoder_buffer_in_4.bin,${transformer_data_path}/encoder_buffer_in_2.bin,${transformer_data_path}/encoder_buffer_in_3.bin,${transformer_data_path}/encoder_buffer_in_7.bin,${transformer_data_path}/encoder_buffer_in_5.bin,${transformer_data_path}/encoder_buffer_in_6.bin,${transformer_data_path}/encoder_buffer_in_10.bin,${transformer_data_path}/encoder_buffer_in_8.bin,${transformer_data_path}/encoder_buffer_in_9.bin,${transformer_data_path}/encoder_buffer_in_11.bin,${transformer_data_path}/encoder_buffer_in_12.bin,${transformer_data_path}/encoder_buffer_in_15.bin,${transformer_data_path}/encoder_buffer_in_13.bin,${transformer_data_path}/encoder_buffer_in_14.bin,${transformer_data_path}/encoder_buffer_in_18.bin,${transformer_data_path}/encoder_buffer_in_16.bin,${transformer_data_path}/encoder_buffer_in_17.bin,${transformer_data_path}/encoder_buffer_in_21.bin,${transformer_data_path}/encoder_buffer_in_19.bin,${transformer_data_path}/encoder_buffer_in_20.bin,${transformer_data_path}/encoder_buffer_in_22.bin,${transformer_data_path}/encoder_buffer_in_23.bin,${transformer_data_path}/encoder_buffer_in_26.bin,${transformer_data_path}/encoder_buffer_in_24.bin,${transformer_data_path}/encoder_buffer_in_25.bin,${transformer_data_path}/encoder_buffer_in_29.bin,${transformer_data_path}/encoder_buffer_in_27.bin,${transformer_data_path}/encoder_buffer_in_28.bin,${transformer_data_path}/encoder_buffer_in_32.bin,${transformer_data_path}/encoder_buffer_in_30.bin,${transformer_data_path}/encoder_buffer_in_31.bin,${transformer_data_path}/encoder_buffer_in_33.bin,${transformer_data_path}/encoder_buffer_in_34.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  551.         fi

  552.         if [[ $model_name == "transformer_20200831_decoder_fp32.tflite" ]]; then

  553.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=${transformer_data_path}/decoder_buffer_in_0-10.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  554.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${transformer_data_path}/decoder_buffer_in_9.bin,${transformer_data_path}/decoder_buffer_in_2.bin,${transformer_data_path}/decoder_buffer_in_0.bin,${transformer_data_path}/decoder_buffer_in_1.bin,${transformer_data_path}/decoder_buffer_in_5.bin,${transformer_data_path}/decoder_buffer_in_3.bin,${transformer_data_path}/decoder_buffer_in_4.bin,${transformer_data_path}/decoder_buffer_in_8.bin,${transformer_data_path}/decoder_buffer_in_6.bin,${transformer_data_path}/decoder_buffer_in_7.bin,${transformer_data_path}/decoder_buffer_in_10.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  555.         fi

  556.         if [ $? = 0 ]; then

  557.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  558.         else

  559.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  560.         fi

  561.     done < ${models_tflite_posttraining_config}

  562. 

  563.     # Run caffe post training quantization converted models:

  564.     while read line; do

  565.         posttraining_line_info=${line}

  566.         if [[ $posttraining_line_info == \#* ]]; then

  567.           continue

  568.         fi

  569.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  570.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  571.         echo ${model_name} >> "${run_x86_log_file}"

  572.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'_posttraining.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out'  --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  573.         ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${models_path}/input_output/input/${model_name}_posttraining.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  574.         if [ $? = 0 ]; then

  575.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  576.         else

  577.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  578.         fi

  579.     done < ${models_caffe_posttraining_config}

  580. 

  581.     # Run tflite aware training quantization converted models:

  582.     while read line; do

  583.         model_name=${line}

  584.         if [[ $model_name == \#* ]]; then

  585.           continue

  586.         fi

  587.         echo ${model_name} >> "${run_x86_log_file}"

  588.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  589.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_log_file}"

  590.         if [ $? = 0 ]; then

  591.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  592.         else

  593.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  594.         fi

  595.     done < ${models_tflite_awaretraining_config}

  596. 

  597.     # Run tflite weight quantization converted models:

  598.     while read line; do

  599.         weight_quant_line_info=${line}

  600.         if [[ $weight_quant_line_info == \#* ]]; then

  601.           continue

  602.         fi

  603.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  604.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  605.         echo ${model_name} >> "${run_x86_log_file}"

  606.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_log_file}"

  607.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit}>> "${run_x86_log_file}"

  608.         if [ $? = 0 ]; then

  609.             run_result='x86: '${model_name}_weightquant' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  610.         else

  611.             run_result='x86: '${model_name}_weightquant' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  612.         fi

  613.     done < ${models_tflite_weightquant_config}

  614. 

  615.     # Run caffe weightquant converted models:

  616.     while read line; do

  617.         weight_quant_line_info=${line}

  618.         if [[ ${weight_quant_line_info} == \#* ]]; then

  619.           continue

  620.         fi

  621.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  622.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  623.         echo ${model_name} >> "${run_x86_log_file}"

  624.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_log_file}"

  625.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit}>> "${run_x86_log_file}"

  626.         if [ $? = 0 ]; then

  627.             run_result='x86: '${model_name}_weightquant' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  628.         else

  629.             run_result='x86: '${model_name}_weightquant' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  630.         fi

  631.     done < ${models_caffe_weightquant_config}

  632. 

  633.     # Run tf weightquant converted models:

  634.     while read line; do

  635.         weight_quant_line_info=${line}

  636.         if [[ $weight_quant_line_info == \#* ]]; then

  637.           continue

  638.         fi

  639.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  640.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  641.         echo ${model_name} >> "${run_x86_log_file}"

  642.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}"

  643.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit}>> "${run_x86_log_file}"

  644.         if [ $? = 0 ]; then

  645.             run_result='x86: '${model_name}_weightquant' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  646.         else

  647.             run_result='x86: '${model_name}_weightquant' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  648.         fi

  649.     done < ${models_tf_weightquant_config}

  650. 

  651.     # Run mindir weight quantization converted models:

  652.     while read line; do

  653.         weight_quant_line_info=${line}

  654.         if [[ $weight_quant_line_info == \#* ]]; then

  655.           continue

  656.         fi

  657.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  658.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  659.         echo ${model_name} >> "${run_x86_log_file}"

  660.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_log_file}"

  661.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}"

  662.         if [ $? = 0 ]; then

  663.             run_result='x86: '${model_name}'_weightquant pass'; echo ${run_result} >> ${run_benchmark_result_file}

  664.         else

  665.             run_result='x86: '${model_name}'_weightquant failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  666.         fi

  667.     done < ${models_mindspore_weightquant_config}

  668. 

  669.     # Run mindir mixbit weight quantization converted models:

  670.     while read line; do

  671.         line_info=${line}

  672.         if [[ $line_info == \#* ]]; then

  673.           continue

  674.         fi

  675.         model_name=`echo ${line_info}|awk -F ' ' '{print $1}'`

  676.         accuracy_limit_7bit=`echo ${line_info}|awk -F ' ' '{print $2}'`

  677.         accuracy_limit_9bit=`echo ${line_info}|awk -F ' ' '{print $3}'`

  678. 

  679.         echo ${model_name} >> "${run_x86_log_file}"

  680.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_7bit.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit_7bit}' >> "${run_x86_log_file}"

  681.         ./benchmark --modelFile=${ms_models_path}/${model_name}_7bit.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit_7bit} >> "${run_x86_log_file}"

  682.         if [ $? = 0 ]; then

  683.             run_result='x86: '${model_name}'_7bit pass'; echo ${run_result} >> ${run_benchmark_result_file}

  684.         else

  685.             run_result='x86: '${model_name}'_7bit failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  686.         fi

  687.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_9bit.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit_9bit}' >> "${run_x86_log_file}"

  688.         ./benchmark --modelFile=${ms_models_path}/${model_name}_9bit.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit_9bit} >> "${run_x86_log_file}"

  689.         if [ $? = 0 ]; then

  690.             run_result='x86: '${model_name}'_9bit pass'; echo ${run_result} >> ${run_benchmark_result_file}

  691.         else

  692.             run_result='x86: '${model_name}'_9bit failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  693.         fi

  694.     done < ${models_mindspore_mixbit_config}

  695. 

  696.     # Run converted models which has multiple inputs:

  697.     while read line; do

  698.         if [[ $line == \#* ]]; then

  699.           continue

  700.         fi

  701.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  702.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  703.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  704.         input_files=''

  705.         output_file=''

  706.         data_path="${models_path}/input_output/"

  707.         for i in $(seq 1 $input_num)

  708.         do

  709.           input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  710.         done

  711.         output_file=${data_path}'output/'${model_name}'.ms.out'

  712.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  713.           model_name=${model_name%.*}

  714.         fi

  715.         echo ${model_name} >> "${run_x86_log_file}"

  716.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_x86_log_file}"

  717.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} >> "${run_x86_log_file}"

  718.         if [ $? = 0 ]; then

  719.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  720.         else

  721.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  722.         fi

  723.     done < ${models_with_multiple_inputs_config}

  724. 

  725.     # Run converted models which does not need to be cared about the accuracy:

  726.     while read line; do

  727.         if [[ $line == \#* ]]; then

  728.           continue

  729.         fi

  730.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  731.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  732.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  733.         input_files=''

  734.         output_file=''

  735.         data_path="${models_path}/input_output/"

  736.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  737.           model_name=${model_name%.*}

  738.         fi

  739.         echo ${model_name} >> "${run_x86_log_file}"

  740.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inputShapes='${input_shapes}' --loopCount=2 --warmUpLoopCount=0' >> "${run_x86_log_file}"

  741.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inputShapes=${input_shapes} --loopCount=2 --warmUpLoopCount=0 >> "${run_x86_log_file}"

  742.         if [ $? = 0 ]; then

  743.             run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  744.         else

  745.             run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  746.         fi

  747.     done < ${models_for_process_only_config}

  748. }

  749. 

  750. # Run on x86 sse platform:

  751. function Run_x86_sse() {

  752.     cd ${x86_path}/sse || exit 1

  753.     tar -zxf mindspore-lite-${version}-inference-linux-x64.tar.gz || exit 1

  754.     cd ${x86_path}/sse/mindspore-lite-${version}-inference-linux-x64 || return 1

  755.     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./inference/lib

  756.     cp tools/benchmark/benchmark ./ || exit 1

  757. 

  758.     # Run tflite converted models:

  759.     while read line; do

  760.         model_name=${line}

  761.         if [[ $model_name == \#* ]]; then

  762.           continue

  763.         fi

  764.         echo ${model_name} >> "${run_x86_sse_log_file}"

  765.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_sse_log_file}"

  766.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_sse_log_file}"

  767.         if [ $? = 0 ]; then

  768.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  769.         else

  770.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  771.         fi

  772.     done < ${models_tflite_config}

  773. 

  774.     # Run caffe converted models:

  775.     while read line; do

  776.         model_name=${line}

  777.         if [[ $model_name == \#* ]]; then

  778.           continue

  779.         fi

  780.         echo ${model_name} >> "${run_x86_sse_log_file}"

  781.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_sse_log_file}"

  782.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_sse_log_file}"

  783.         if [ $? = 0 ]; then

  784.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  785.         else

  786.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  787.         fi

  788.     done < ${models_caffe_config}

  789. 

  790.     # Run onnx converted models:

  791.     while read line; do

  792.         model_name=${line%;*}

  793.         length=${#model_name}

  794.         input_shapes=${line:length+1}

  795.         if [[ $model_name == \#* ]]; then

  796.           continue

  797.         fi

  798.         echo ${model_name} >> "${run_x86_sse_log_file}"

  799.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_sse_log_file}"

  800.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --inputShapes=${input_shapes} --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_sse_log_file}"

  801.         if [ $? = 0 ]; then

  802.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  803.         else

  804.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  805.         fi

  806.     done < ${models_onnx_config}

  807. 

  808.     # Run mindspore converted models:

  809.     while read line; do

  810.         mindspore_line_info=${line}

  811.         if [[ $mindspore_line_info == \#* ]]; then

  812.           continue

  813.         fi

  814.         model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'`

  815.         accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'`

  816.         echo "---------------------------------------------------------" >> "${run_x86_sse_log_file}"

  817.         echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_x86_sse_log_file}"

  818.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_sse_log_file}"

  819.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  820.         if [ $? = 0 ]; then

  821.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  822.         else

  823.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  824.         fi

  825.     done < ${models_mindspore_config}

  826. 

  827.     # Run mindspore converted train models:

  828.     while read line; do

  829.         model_name=${line}

  830.         if [[ $model_name == \#* ]]; then

  831.           continue

  832.         fi

  833.         echo ${model_name}'_train' >> "${run_x86_sse_log_file}"

  834.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_train.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=${models_path}/input_output/output/'${model_name}'.train.ms.out' >> "${run_x86_sse_log_file}"

  835.         ./benchmark --modelFile=${ms_models_path}/${model_name}'_train'.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.train.ms.out --accuracyThreshold=1.5 >> "${run_x86_sse_log_file}"

  836.         if [ $? = 0 ]; then

  837.             run_result='x86_sse: '${model_name}'_train pass'; echo ${run_result} >> ${run_benchmark_result_file}

  838.         else

  839.             run_result='x86_sse: '${model_name}'_train failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  840.         fi

  841.     done < ${models_mindspore_train_config}

  842. 

  843.     # Run tflite post training quantization converted models:

  844.     while read line; do

  845.         posttraining_line_info=${line}

  846.         if [[ $posttraining_line_info == \#* ]]; then

  847.           continue

  848.         fi

  849.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  850.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  851.         transformer_data_path="${models_path}/input_output/input"

  852.         echo ${model_name} >> "${run_x86_sse_log_file}"

  853.         if [[ $model_name == "mobilenet.tflite" ]]; then

  854.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  855.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  856.         fi

  857.         if [[ $model_name == "transformer_20200831_encoder_fp32.tflite" ]]; then

  858.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_0-35.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  859.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_35.bin,${transformer_data_path}/encoder_buffer_in_0.bin,${transformer_data_path}/encoder_buffer_in_1.bin,${transformer_data_path}/encoder_buffer_in_4.bin,${transformer_data_path}/encoder_buffer_in_2.bin,${transformer_data_path}/encoder_buffer_in_3.bin,${transformer_data_path}/encoder_buffer_in_7.bin,${transformer_data_path}/encoder_buffer_in_5.bin,${transformer_data_path}/encoder_buffer_in_6.bin,${transformer_data_path}/encoder_buffer_in_10.bin,${transformer_data_path}/encoder_buffer_in_8.bin,${transformer_data_path}/encoder_buffer_in_9.bin,${transformer_data_path}/encoder_buffer_in_11.bin,${transformer_data_path}/encoder_buffer_in_12.bin,${transformer_data_path}/encoder_buffer_in_15.bin,${transformer_data_path}/encoder_buffer_in_13.bin,${transformer_data_path}/encoder_buffer_in_14.bin,${transformer_data_path}/encoder_buffer_in_18.bin,${transformer_data_path}/encoder_buffer_in_16.bin,${transformer_data_path}/encoder_buffer_in_17.bin,${transformer_data_path}/encoder_buffer_in_21.bin,${transformer_data_path}/encoder_buffer_in_19.bin,${transformer_data_path}/encoder_buffer_in_20.bin,${transformer_data_path}/encoder_buffer_in_22.bin,${transformer_data_path}/encoder_buffer_in_23.bin,${transformer_data_path}/encoder_buffer_in_26.bin,${transformer_data_path}/encoder_buffer_in_24.bin,${transformer_data_path}/encoder_buffer_in_25.bin,${transformer_data_path}/encoder_buffer_in_29.bin,${transformer_data_path}/encoder_buffer_in_27.bin,${transformer_data_path}/encoder_buffer_in_28.bin,${transformer_data_path}/encoder_buffer_in_32.bin,${transformer_data_path}/encoder_buffer_in_30.bin,${transformer_data_path}/encoder_buffer_in_31.bin,${transformer_data_path}/encoder_buffer_in_33.bin,${transformer_data_path}/encoder_buffer_in_34.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  860.         fi

  861.         if [[ $model_name == "transformer_20200831_decoder_fp32.tflite" ]]; then

  862.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_0-10.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  863.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${transformer_data_path}/decoder_buffer_in_9.bin,${transformer_data_path}/decoder_buffer_in_2.bin,${transformer_data_path}/decoder_buffer_in_0.bin,${transformer_data_path}/decoder_buffer_in_1.bin,${transformer_data_path}/decoder_buffer_in_5.bin,${transformer_data_path}/decoder_buffer_in_3.bin,${transformer_data_path}/decoder_buffer_in_4.bin,${transformer_data_path}/decoder_buffer_in_8.bin,${transformer_data_path}/decoder_buffer_in_6.bin,${transformer_data_path}/decoder_buffer_in_7.bin,${transformer_data_path}/decoder_buffer_in_10.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  864.         fi

  865.         if [ $? = 0 ]; then

  866.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  867.         else

  868.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  869.         fi

  870.     done < ${models_tflite_posttraining_config}

  871. 

  872.     # Run caffe post training quantization converted models:

  873.     while read line; do

  874.         posttraining_line_info=${line}

  875.         if [[ $posttraining_line_info == \#* ]]; then

  876.           continue

  877.         fi

  878.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  879.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  880.         echo ${model_name} >> "${run_x86_sse_log_file}"

  881.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'_posttraining.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out'  --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  882.         ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${models_path}/input_output/input/${model_name}_posttraining.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  883.         if [ $? = 0 ]; then

  884.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  885.         else

  886.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  887.         fi

  888.     done < ${models_caffe_posttraining_config}

  889. 

  890.     # Run tflite aware training quantization converted models:

  891.     while read line; do

  892.         model_name=${line}

  893.         if [[ $model_name == \#* ]]; then

  894.           continue

  895.         fi

  896.         echo ${model_name} >> "${run_x86_sse_log_file}"

  897.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_sse_log_file}"

  898.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_sse_log_file}"

  899.         if [ $? = 0 ]; then

  900.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  901.         else

  902.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  903.         fi

  904.     done < ${models_tflite_awaretraining_config}

  905. 

  906.     # Run tflite weight quantization converted models:

  907.     while read line; do

  908.         weight_quant_line_info=${line}

  909.         if [[ $weight_quant_line_info == \#* ]]; then

  910.           continue

  911.         fi

  912.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  913.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  914.         echo ${model_name} >> "${run_x86_sse_log_file}"

  915.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_sse_log_file}"

  916.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  917.         if [ $? = 0 ]; then

  918.             run_result='x86_sse: '${model_name}_weightquant' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  919.         else

  920.             run_result='x86_sse: '${model_name}_weightquant' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  921.         fi

  922.     done < ${models_tflite_weightquant_config}

  923. 

  924.     # Run mindir weight quantization converted models:

  925.     while read line; do

  926.         weight_quant_line_info=${line}

  927.         if [[ $weight_quant_line_info == \#* ]]; then

  928.           continue

  929.         fi

  930.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  931.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  932.         echo ${model_name} >> "${run_x86_sse_log_file}"

  933.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_sse_log_file}"

  934.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}"

  935.         if [ $? = 0 ]; then

  936.             run_result='x86_sse: '${model_name}'_weightquant pass'; echo ${run_result} >> ${run_benchmark_result_file}

  937.         else

  938.             run_result='x86_sse: '${model_name}'_weightquant failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  939.         fi

  940.     done < ${models_mindspore_weightquant_config}

  941. 

  942.     # Run mindir mixbit weight quantization converted models:

  943.     while read line; do

  944.         line_info=${line}

  945.         if [[ $line_info == \#* ]]; then

  946.           continue

  947.         fi

  948.         model_name=`echo ${line_info}|awk -F ' ' '{print $1}'`

  949.         accuracy_limit_7bit=`echo ${line_info}|awk -F ' ' '{print $2}'`

  950.         accuracy_limit_9bit=`echo ${line_info}|awk -F ' ' '{print $3}'`

  951. 

  952.         echo ${model_name} >> "${run_x86_sse_log_file}"

  953.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_7bit.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit_7bit}' >> "${run_x86_sse_log_file}"

  954.         ./benchmark --modelFile=${ms_models_path}/${model_name}_7bit.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit_7bit} >> "${run_x86_sse_log_file}"

  955.         if [ $? = 0 ]; then

  956.             run_result='x86_sse: '${model_name}'_7bit pass'; echo ${run_result} >> ${run_benchmark_result_file}

  957.         else

  958.             run_result='x86_sse: '${model_name}'_7bit failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  959.         fi

  960.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_9bit.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit_9bit}' >> "${run_x86_sse_log_file}"

  961.         ./benchmark --modelFile=${ms_models_path}/${model_name}_9bit.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit_9bit} >> "${run_x86_sse_log_file}"

  962.         if [ $? = 0 ]; then

  963.             run_result='x86_sse: '${model_name}'_9bit pass'; echo ${run_result} >> ${run_benchmark_result_file}

  964.         else

  965.             run_result='x86_sse: '${model_name}'_9bit failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  966.         fi

  967.     done < ${models_mindspore_mixbit_config}

  968. 

  969.     # Run converted models with multiple inputs:

  970.     while read line; do

  971.         model_name=${line%%;*}

  972.         if [[ $model_name == \#* ]]; then

  973.           continue

  974.         fi

  975.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  976.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  977.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  978.         input_files=''

  979.         output_file=''

  980.         data_path="${models_path}/input_output/"

  981.         for i in $(seq 1 $input_num)

  982.         do

  983.           input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  984.         done

  985.         output_file=${data_path}'output/'${model_name}'.ms.out'

  986.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  987.           model_name=${model_name%.*}

  988.         fi

  989.         echo ${model_name} >> "${run_x86_sse_log_file}"

  990.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_x86_sse_log_file}"

  991.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} >> "${run_x86_sse_log_file}"

  992.         if [ $? = 0 ]; then

  993.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  994.         else

  995.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  996.         fi

  997.     done < ${models_with_multiple_inputs_config}

  998. 

  999.     # Run converted models which does not need to be cared about the accuracy:

  1000.     while read line; do

  1001.         model_name=${line%%;*}

  1002.         if [[ $model_name == \#* ]]; then

  1003.           continue

  1004.         fi

  1005.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  1006.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  1007.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  1008.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  1009.           model_name=${model_name%.*}

  1010.         fi

  1011.         echo ${model_name} >> "${run_x86_sse_log_file}"

  1012.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inputShapes='${input_shapes}' --loopCount=2 --warmUpLoopCount=0' >> "${run_x86_sse_log_file}"

  1013.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inputShapes=${input_shapes} --loopCount=2 --warmUpLoopCount=0 >> "${run_x86_sse_log_file}"

  1014.         if [ $? = 0 ]; then

  1015.             run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1016.         else

  1017.             run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1018.         fi

  1019.     done < ${models_for_process_only_config}

  1020. }

  1021. 

  1022. # Run on x86 avx platform:

  1023. function Run_x86_avx() {

  1024.     cd ${x86_path}/avx || exit 1

  1025.     tar -zxf mindspore-lite-${version}-inference-linux-x64.tar.gz || exit 1

  1026.     cd ${x86_path}/avx/mindspore-lite-${version}-inference-linux-x64 || return 1

  1027.     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./inference/lib

  1028.     cp tools/benchmark/benchmark ./ || exit 1

  1029. 

  1030.     # Run tflite converted models:

  1031.     while read line; do

  1032.         model_name=${line}

  1033.         if [[ $model_name == \#* ]]; then

  1034.           continue

  1035.         fi

  1036.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1037.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_avx_log_file}"

  1038.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_avx_log_file}"

  1039.         if [ $? = 0 ]; then

  1040.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1041.         else

  1042.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1043.         fi

  1044.     done < ${models_tflite_config}

  1045. 

  1046.     # Run caffe converted models:

  1047.     while read line; do

  1048.         model_name=${line}

  1049.         if [[ $model_name == \#* ]]; then

  1050.           continue

  1051.         fi

  1052.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1053.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_avx_log_file}"

  1054.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_avx_log_file}"

  1055.         if [ $? = 0 ]; then

  1056.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1057.         else

  1058.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1059.         fi

  1060.     done < ${models_caffe_config}

  1061. 

  1062.     # Run onnx converted models:

  1063.     while read line; do

  1064.         model_name=${line%;*}

  1065.         length=${#model_name}

  1066.         input_shapes=${line:length+1}

  1067.         if [[ $model_name == \#* ]]; then

  1068.           continue

  1069.         fi

  1070.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1071.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_avx_log_file}"

  1072.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --inputShapes=${input_shapes} --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_avx_log_file}"

  1073.         if [ $? = 0 ]; then

  1074.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1075.         else

  1076.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1077.         fi

  1078.     done < ${models_onnx_config}

  1079. 

  1080.     # Run mindspore converted models:

  1081.     while read line; do

  1082.         mindspore_line_info=${line}

  1083.         if [[ $mindspore_line_info == \#* ]]; then

  1084.           continue

  1085.         fi

  1086.         model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'`

  1087.         accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'`

  1088.         echo "---------------------------------------------------------" >> "${run_x86_avx_log_file}"

  1089.         echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_x86_avx_log_file}"

  1090.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_avx_log_file}"

  1091.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1092.         if [ $? = 0 ]; then

  1093.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1094.         else

  1095.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1096.         fi

  1097.     done < ${models_mindspore_config}

  1098. 

  1099.     # Run mindspore converted train models:

  1100.     while read line; do

  1101.         model_name=${line}

  1102.         if [[ $model_name == \#* ]]; then

  1103.           continue

  1104.         fi

  1105.         echo ${model_name}'_train' >> "${run_x86_avx_log_file}"

  1106.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_train.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=${models_path}/input_output/output/'${model_name}'.train.ms.out' >> "${run_x86_avx_log_file}"

  1107.         ./benchmark --modelFile=${ms_models_path}/${model_name}'_train'.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.train.ms.out --accuracyThreshold=1.5 >> "${run_x86_avx_log_file}"

  1108.         if [ $? = 0 ]; then

  1109.             run_result='x86_avx: '${model_name}'_train pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1110.         else

  1111.             run_result='x86_avx: '${model_name}'_train failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1112.         fi

  1113.     done < ${models_mindspore_train_config}

  1114. 

  1115.     # Run tflite post training quantization converted models:

  1116.     while read line; do

  1117.         posttraining_line_info=${line}

  1118.         if [[ $posttraining_line_info == \#* ]]; then

  1119.           continue

  1120.         fi

  1121.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  1122.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  1123.         transformer_data_path="${models_path}/input_output/input"

  1124.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1125.         if [[ $model_name == "mobilenet.tflite" ]]; then

  1126.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1127.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/quantTraining/mnist_calibration_data/00099.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1128.         fi

  1129.         if [[ $model_name == "transformer_20200831_encoder_fp32.tflite" ]]; then

  1130.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_0-35.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1131.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_35.bin,${transformer_data_path}/encoder_buffer_in_0.bin,${transformer_data_path}/encoder_buffer_in_1.bin,${transformer_data_path}/encoder_buffer_in_4.bin,${transformer_data_path}/encoder_buffer_in_2.bin,${transformer_data_path}/encoder_buffer_in_3.bin,${transformer_data_path}/encoder_buffer_in_7.bin,${transformer_data_path}/encoder_buffer_in_5.bin,${transformer_data_path}/encoder_buffer_in_6.bin,${transformer_data_path}/encoder_buffer_in_10.bin,${transformer_data_path}/encoder_buffer_in_8.bin,${transformer_data_path}/encoder_buffer_in_9.bin,${transformer_data_path}/encoder_buffer_in_11.bin,${transformer_data_path}/encoder_buffer_in_12.bin,${transformer_data_path}/encoder_buffer_in_15.bin,${transformer_data_path}/encoder_buffer_in_13.bin,${transformer_data_path}/encoder_buffer_in_14.bin,${transformer_data_path}/encoder_buffer_in_18.bin,${transformer_data_path}/encoder_buffer_in_16.bin,${transformer_data_path}/encoder_buffer_in_17.bin,${transformer_data_path}/encoder_buffer_in_21.bin,${transformer_data_path}/encoder_buffer_in_19.bin,${transformer_data_path}/encoder_buffer_in_20.bin,${transformer_data_path}/encoder_buffer_in_22.bin,${transformer_data_path}/encoder_buffer_in_23.bin,${transformer_data_path}/encoder_buffer_in_26.bin,${transformer_data_path}/encoder_buffer_in_24.bin,${transformer_data_path}/encoder_buffer_in_25.bin,${transformer_data_path}/encoder_buffer_in_29.bin,${transformer_data_path}/encoder_buffer_in_27.bin,${transformer_data_path}/encoder_buffer_in_28.bin,${transformer_data_path}/encoder_buffer_in_32.bin,${transformer_data_path}/encoder_buffer_in_30.bin,${transformer_data_path}/encoder_buffer_in_31.bin,${transformer_data_path}/encoder_buffer_in_33.bin,${transformer_data_path}/encoder_buffer_in_34.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1132.         fi

  1133.         if [[ $model_name == "transformer_20200831_decoder_fp32.tflite" ]]; then

  1134.             echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile=${transformer_data_path}/encoder_buffer_in_0-10.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1135.             ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${transformer_data_path}/decoder_buffer_in_9.bin,${transformer_data_path}/decoder_buffer_in_2.bin,${transformer_data_path}/decoder_buffer_in_0.bin,${transformer_data_path}/decoder_buffer_in_1.bin,${transformer_data_path}/decoder_buffer_in_5.bin,${transformer_data_path}/decoder_buffer_in_3.bin,${transformer_data_path}/decoder_buffer_in_4.bin,${transformer_data_path}/decoder_buffer_in_8.bin,${transformer_data_path}/decoder_buffer_in_6.bin,${transformer_data_path}/decoder_buffer_in_7.bin,${transformer_data_path}/decoder_buffer_in_10.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1136.         fi

  1137.         if [ $? = 0 ]; then

  1138.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1139.         else

  1140.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1141.         fi

  1142.     done < ${models_tflite_posttraining_config}

  1143. 

  1144.     # Run caffe post training quantization converted models:

  1145.     while read line; do

  1146.         posttraining_line_info=${line}

  1147.         if [[ $posttraining_line_info == \#* ]]; then

  1148.           continue

  1149.         fi

  1150.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  1151.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  1152.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1153.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_posttraining.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'_posttraining.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'_posttraining.ms.out'  --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1154.         ./benchmark --modelFile=${ms_models_path}/${model_name}_posttraining.ms --inDataFile=${models_path}/input_output/input/${model_name}_posttraining.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}_posttraining.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1155.         if [ $? = 0 ]; then

  1156.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1157.         else

  1158.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1159.         fi

  1160.     done < ${models_caffe_posttraining_config}

  1161. 

  1162.     # Run tflite aware training quantization converted models:

  1163.     while read line; do

  1164.         model_name=${line}

  1165.         if [[ $model_name == \#* ]]; then

  1166.           continue

  1167.         fi

  1168.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1169.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out' >> "${run_x86_avx_log_file}"

  1170.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out >> "${run_x86_avx_log_file}"

  1171.         if [ $? = 0 ]; then

  1172.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1173.         else

  1174.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1175.         fi

  1176.     done < ${models_tflite_awaretraining_config}

  1177. 

  1178.     # Run tflite weight quantization converted models:

  1179.     while read line; do

  1180.         weight_quant_line_info=${line}

  1181.         if [[ $weight_quant_line_info == \#* ]]; then

  1182.           continue

  1183.         fi

  1184.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  1185.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  1186.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1187.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_avx_log_file}"

  1188.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1189.         if [ $? = 0 ]; then

  1190.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1191.         else

  1192.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1193.         fi

  1194.     done < ${models_tflite_weightquant_config}

  1195. 

  1196.     # Run mindir weight quantization converted models:

  1197.     while read line; do

  1198.         weight_quant_line_info=${line}

  1199.         if [[ $weight_quant_line_info == \#* ]]; then

  1200.           continue

  1201.         fi

  1202.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  1203.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  1204.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1205.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit}' >> "${run_x86_avx_log_file}"

  1206.         ./benchmark --modelFile=${ms_models_path}/${model_name}_weightquant.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_avx_log_file}"

  1207.         if [ $? = 0 ]; then

  1208.             run_result='x86_avx: '${model_name}'_weightquant pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1209.         else

  1210.             run_result='x86_avx: '${model_name}'_weightquant failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1211.         fi

  1212.     done < ${models_mindspore_weightquant_config}

  1213. 

  1214.     # Run mindir mixbit weight quantization converted models:

  1215.     while read line; do

  1216.         line_info=${line}

  1217.         if [[ $line_info == \#* ]]; then

  1218.           continue

  1219.         fi

  1220.         model_name=`echo ${line_info}|awk -F ' ' '{print $1}'`

  1221.         accuracy_limit_7bit=`echo ${line_info}|awk -F ' ' '{print $2}'`

  1222.         accuracy_limit_9bit=`echo ${line_info}|awk -F ' ' '{print $3}'`

  1223. 

  1224.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1225.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_7bit.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit_7bit}' >> "${run_x86_avx_log_file}"

  1226.         ./benchmark --modelFile=${ms_models_path}/${model_name}_7bit.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit_7bit} >> "${run_x86_avx_log_file}"

  1227.         if [ $? = 0 ]; then

  1228.             run_result='x86_avx: '${model_name}'_7bit pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1229.         else

  1230.             run_result='x86_avx: '${model_name}'_7bit failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1231.         fi

  1232.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'_9bit.ms --inDataFile='${models_path}'/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile='${models_path}'/input_output/output/'${model_name}'.ms.out --accuracyThreshold=${accuracy_limit_9bit}' >> "${run_x86_avx_log_file}"

  1233.         ./benchmark --modelFile=${ms_models_path}/${model_name}_9bit.ms --inDataFile=${models_path}/input_output/input/${model_name}.ms.bin --benchmarkDataFile=${models_path}/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit_9bit} >> "${run_x86_avx_log_file}"

  1234.         if [ $? = 0 ]; then

  1235.             run_result='x86_avx: '${model_name}'_9bit pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1236.         else

  1237.             run_result='x86_avx: '${model_name}'_9bit failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1238.         fi

  1239.     done < ${models_mindspore_mixbit_config}

  1240. 

  1241.     # Run converted models which has multiple inputs:

  1242.     while read line; do

  1243.         model_name=${line%%;*}

  1244.         if [[ $model_name == \#* ]]; then

  1245.           continue

  1246.         fi

  1247.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  1248.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  1249.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  1250.         input_files=''

  1251.         output_file=''

  1252.         data_path="${models_path}/input_output/"

  1253.         for i in $(seq 1 $input_num)

  1254.         do

  1255.           input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  1256.         done

  1257.         output_file=${data_path}'output/'${model_name}'.ms.out'

  1258.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  1259.           model_name=${model_name%.*}

  1260.         fi

  1261.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1262.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_x86_avx_log_file}"

  1263.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} >> "${run_x86_avx_log_file}"

  1264.         if [ $? = 0 ]; then

  1265.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1266.         else

  1267.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1268.         fi

  1269.     done < ${models_with_multiple_inputs_config}

  1270. 

  1271.     # Run converted models which does not need to be cared about the accuracy:

  1272.     while read line; do

  1273.         model_name=${line%%;*}

  1274.         if [[ $model_name == \#* ]]; then

  1275.           continue

  1276.         fi

  1277.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  1278.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  1279.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  1280.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  1281.           model_name=${model_name%.*}

  1282.         fi

  1283.         echo ${model_name} >> "${run_x86_avx_log_file}"

  1284.         echo './benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inputShapes='${input_shapes}' --loopCount=2 --warmUpLoopCount=0' >> "${run_x86_avx_log_file}"

  1285.         ./benchmark --modelFile=${ms_models_path}/${model_name}.ms --inputShapes=${input_shapes} --loopCount=2 --warmUpLoopCount=0 >> "${run_x86_avx_log_file}"

  1286.         if [ $? = 0 ]; then

  1287.             run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1288.         else

  1289.             run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1290.         fi

  1291.     done < ${models_for_process_only_config}

  1292. }

  1293. 

  1294. function Run_arm64_codegen() {

  1295.     echo "ANDROID_NDK: ${ANDROID_NDK}" >> ${run_arm64_fp32_codegen_log_file}

  1296.     cd ${arm64_path} || exit 1

  1297.     tar -zxf mindspore-lite-${version}-inference-android-aarch64.tar.gz || exit 1

  1298.     local PKG_PATH=${arm64_path}/mindspore-lite-${version}-inference-android-aarch64

  1299.     local CODEGEN_PATH=${x86_path}/mindspore-lite-${version}-inference-linux-x64/tools/codegen

  1300. 

  1301.     rm -rf ${build_path}

  1302.     mkdir -p ${build_path}

  1303. 

  1304.     # Run tflite converted models:

  1305.     while read line; do

  1306.         model_name=${line}

  1307.         if [[ $model_name == \#* ]]; then

  1308.           continue

  1309.         fi

  1310. 

  1311.         {

  1312.             echo "arm64_codegen: ${model_name}"

  1313.             echo "${CODEGEN_PATH}/codegen --codePath=${build_path} --modelPath=${ms_models_path}/${model_name}.ms --target=ARM64"

  1314.             ${CODEGEN_PATH}/codegen --codePath=${build_path} --modelPath=${ms_models_path}/${model_name}.ms --target=ARM64

  1315.         } >> ${run_arm64_fp32_codegen_log_file}

  1316. 

  1317.         rm -rf ${build_path}/benchmark

  1318.         mkdir -p ${build_path}/benchmark && cd ${build_path}/benchmark || exit 1

  1319. 

  1320.         {

  1321.             echo "cmake -DCMAKE_BUILD_TYPE=Release

  1322.                   -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake

  1323.                   -DANDROID_ABI=arm64-v8a

  1324.                   -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang

  1325.                   -DANDROID_NATIVE_API_LEVEL=19

  1326.                   -DPLATFORM_ARM64=ON

  1327.                   -DPKG_PATH=${PKG_PATH} ${build_path}/${model_name}"

  1328. 

  1329.             cmake -DCMAKE_BUILD_TYPE=Release \

  1330.                   -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \

  1331.                   -DANDROID_ABI="arm64-v8a" \

  1332.                   -DANDROID_TOOLCHAIN_NAME="aarch64-linux-android-clang" \

  1333.                   -DANDROID_NATIVE_API_LEVEL="19" \

  1334.                   -DPLATFORM_ARM64=ON \

  1335.                   -DPKG_PATH=${PKG_PATH} ${build_path}/${model_name}

  1336. 

  1337.             make -j4

  1338.         } >> ${run_arm64_fp32_codegen_log_file}

  1339. 

  1340.         rm -rf ${build_path}/codegen_test

  1341.         mkdir ${build_path}/codegen_test && cd ${build_path}/codegen_test || exit 1

  1342.         cp -a ${build_path}/benchmark/benchmark ${build_path}/codegen_test/benchmark || exit 1

  1343.         cp -a ${build_path}/${model_name}/src/net.bin ${build_path}/codegen_test/net.bin || exit 1

  1344. 

  1345.         {

  1346.             echo 'ls ${build_path}/codegen_test'

  1347.             ls ${build_path}/codegen_test

  1348.         } >> ${run_arm64_fp32_codegen_log_file}

  1349. 

  1350.         # adb push all needed files to the phone

  1351.         adb -s ${device_id} push ${build_path}/codegen_test /data/local/tmp/ > adb_push_log.txt

  1352. 

  1353.         {

  1354.             echo 'cd  /data/local/tmp/codegen_test'

  1355.             echo 'chmod 777 benchmark'

  1356.             echo 'chmod 777 net.bin'

  1357.             echo 'ls'

  1358.             echo './benchmark /data/local/tmp/input_output/input/'${model_name}'.ms.bin ./net.bin 1 /data/local/tmp/input_output/output/'${model_name}'.ms.out'

  1359.             echo 'cd .. && rm -rf codegen_test'

  1360.         } >> ${run_arm64_fp32_codegen_log_file}

  1361. 

  1362.         {

  1363.             echo 'cd  /data/local/tmp/codegen_test'

  1364.             echo 'chmod 777 benchmark'

  1365.             echo 'chmod 777 net.bin'

  1366.             echo 'ls'

  1367.             echo './benchmark /data/local/tmp/input_output/input/'${model_name}'.ms.bin ./net.bin 1 /data/local/tmp/input_output/output/'${model_name}'.ms.out'

  1368.             echo 'cd .. && rm -rf codegen_test'

  1369.         } > adb_run_cmd.txt

  1370. 

  1371.         adb -s ${device_id} shell < adb_run_cmd.txt >> ${run_arm64_fp32_codegen_log_file}

  1372.         if [ $? = 0 ]; then

  1373.             run_result='arm64_codegen: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1374.         else

  1375.             run_result='arm64_codegen: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1376.         fi

  1377.     done < ${models_codegen_config}

  1378. 

  1379.     rm -rf ${build_path}

  1380. }

  1381. 

  1382. function Run_arm32_codegen() {

  1383.     echo "ANDROID_NDK: ${ANDROID_NDK}" >> ${run_arm32_fp32_codegen_log_file}

  1384.     cd ${arm32_path} || exit 1

  1385.     tar -zxf mindspore-lite-${version}-inference-android-aarch32.tar.gz || exit 1

  1386.     local PKG_PATH=${arm32_path}/mindspore-lite-${version}-inference-android-aarch32

  1387.     local CODEGEN_PATH=${x86_path}/mindspore-lite-${version}-inference-linux-x64/tools/codegen

  1388. 

  1389.     rm -rf ${build_path}

  1390.     mkdir -p ${build_path}

  1391. 

  1392.     # Run tflite converted models:

  1393.     while read line; do

  1394.         model_name=${line}

  1395.         if [[ $model_name == \#* ]]; then

  1396.           continue

  1397.         fi

  1398. 

  1399.         {

  1400.             echo "arm32_codegen: ${model_name}"

  1401.             echo "${CODEGEN_PATH}/codegen --codePath=${build_path} --modelPath=${ms_models_path}/${model_name}.ms --target=ARM32A"

  1402.             ${CODEGEN_PATH}/codegen --codePath=${build_path} --modelPath=${ms_models_path}/${model_name}.ms --target=ARM32A

  1403.         } >> ${run_arm32_fp32_codegen_log_file}

  1404. 

  1405.         rm -rf ${build_path}/benchmark

  1406.         mkdir -p ${build_path}/benchmark && cd ${build_path}/benchmark || exit 1

  1407. 

  1408.         {

  1409.             echo "cmake -DCMAKE_BUILD_TYPE=Release \

  1410.                   -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \

  1411.                   -DANDROID_ABI=armeabi-v7a \

  1412.                   -DANDROID_TOOLCHAIN_NAME=clang \

  1413.                   -DANDROID_NATIVE_API_LEVEL=19 \

  1414.                   -DPLATFORM_ARM32=ON \

  1415.                   -DPKG_PATH=${PKG_PATH} ${build_path}/${model_name}"

  1416. 

  1417.             cmake -DCMAKE_BUILD_TYPE=Release \

  1418.                   -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \

  1419.                   -DANDROID_ABI="armeabi-v7a" \

  1420.                   -DANDROID_TOOLCHAIN_NAME="clang" \

  1421.                   -DANDROID_NATIVE_API_LEVEL="19" \

  1422.                   -DPLATFORM_ARM32=ON \

  1423.                   -DPKG_PATH=${PKG_PATH} ${build_path}/${model_name}

  1424. 

  1425.             make -j4

  1426.         } >> ${run_arm32_fp32_codegen_log_file}

  1427. 

  1428.         rm -rf ${build_path}/codegen_test

  1429.         mkdir ${build_path}/codegen_test && cd ${build_path}/codegen_test || exit 1

  1430.         cp -a ${build_path}/benchmark/benchmark ${build_path}/codegen_test/benchmark || exit 1

  1431.         cp -a ${build_path}/${model_name}/src/net.bin ${build_path}/codegen_test/net.bin || exit 1

  1432. 

  1433.         {

  1434.             echo 'ls ${build_path}/codegen_test'

  1435.             ls ${build_path}/codegen_test

  1436.         } >> ${run_arm32_fp32_codegen_log_file}

  1437. 

  1438.         # adb push all needed files to the phone

  1439.         adb -s ${device_id} push ${build_path}/codegen_test /data/local/tmp/ > adb_push_log.txt

  1440. 

  1441.         {

  1442.             echo 'cd  /data/local/tmp/codegen_test'

  1443.             echo 'chmod 777 benchmark'

  1444.             echo 'chmod 777 net.bin'

  1445.             echo 'ls'

  1446.             echo './benchmark /data/local/tmp/input_output/input/'${model_name}'.ms.bin ./net.bin 1 /data/local/tmp/input_output/output/'${model_name}'.ms.out'

  1447.             echo 'cd .. && rm -rf codegen_test'

  1448.         } >> ${run_arm32_fp32_codegen_log_file}

  1449. 

  1450.         {

  1451.             echo 'cd  /data/local/tmp/codegen_test'

  1452.             echo 'chmod 777 benchmark'

  1453.             echo 'chmod 777 net.bin'

  1454.             echo 'ls'

  1455.             echo './benchmark /data/local/tmp/input_output/input/'${model_name}'.ms.bin ./net.bin 1 /data/local/tmp/input_output/output/'${model_name}'.ms.out'

  1456.             echo 'cd .. && rm -rf codegen_test'

  1457.         } > adb_run_cmd.txt

  1458. 

  1459.         adb -s ${device_id} shell < adb_run_cmd.txt >> ${run_arm32_fp32_codegen_log_file}

  1460.         if [ $? = 0 ]; then

  1461.             run_result='arm32_codegen: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1462.         else

  1463.             run_result='arm32_codegen: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1464.         fi

  1465.     done < ${models_codegen_config}

  1466. 

  1467.     rm -rf ${build_path}

  1468. }

  1469. 

  1470. # Run on arm64 platform:

  1471. function Run_arm64() {

  1472.     cd ${arm64_path} || exit 1

  1473.     tar -zxf mindspore-lite-${version}-inference-android-aarch64.tar.gz || exit 1

  1474. 

  1475.     # If build with minddata, copy the minddata related libs

  1476.     cd ${benchmark_test_path} || exit 1

  1477.     if [ -f ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ]; then

  1478.         cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  1479.     fi

  1480.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai.so ${benchmark_test_path}/libhiai.so || exit 1

  1481.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir.so ${benchmark_test_path}/libhiai_ir.so || exit 1

  1482.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir_build.so ${benchmark_test_path}/libhiai_ir_build.so || exit 1

  1483. 

  1484.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  1485.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/tools/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  1486. 

  1487.     # adb push all needed files to the phone

  1488.     adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  1489. 

  1490.     # run adb ,run session ,check the result:

  1491.     echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  1492.     echo 'cp  /data/local/tmp/libc++_shared.so ./' >> adb_cmd.txt

  1493.     echo 'chmod 777 benchmark' >> adb_cmd.txt

  1494. 

  1495.     adb -s ${device_id} shell < adb_cmd.txt

  1496. 

  1497.     # Run compatibility test models:

  1498.     while read line; do

  1499.         model_name=${line%;*}

  1500.         length=${#model_name}

  1501.         input_shapes=${line:length+1}

  1502.         if [[ $model_name == \#* ]]; then

  1503.           continue

  1504.         fi

  1505.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1506.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1507.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes} >> "${run_arm64_fp32_log_file}"

  1508.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes} >> adb_run_cmd.txt

  1509.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1510.         if [ $? = 0 ]; then

  1511.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1512.         else

  1513.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1514.         fi

  1515.         # run benchmark test without clib data

  1516.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1517.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2 --inputShapes='${input_shapes} >> "${run_arm64_fp32_log_file}"

  1518.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2 --inputShapes='${input_shapes} >> adb_run_cmd.txt

  1519.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1520.         if [ $? = 0 ]; then

  1521.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1522.         else

  1523.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1524.         fi

  1525.     done < ${models_compatibility_config}

  1526. 

  1527.     # Run tf converted models:

  1528.     while read line; do

  1529.         model_name=${line%;*}

  1530.         length=${#model_name}

  1531.         input_shapes=${line:length+1}

  1532.         if [[ $model_name == \#* ]]; then

  1533.           continue

  1534.         fi

  1535.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1536.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1537.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> "${run_arm64_fp32_log_file}"

  1538.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> adb_run_cmd.txt

  1539.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1540.         if [ $? = 0 ]; then

  1541.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1542.         else

  1543.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1544.         fi

  1545.         # run benchmark test without clib data

  1546.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1547.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1548.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=1 --loopCount=2' >> "{run_arm64_fp32_log_file}"

  1549.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1550.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1551.         if [ $? = 0 ]; then

  1552.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1553.         else

  1554.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1555.         fi

  1556.     done < ${models_tf_config}

  1557. 

  1558.     # Run tflite converted models:

  1559.     while read line; do

  1560.         model_name=${line}

  1561.         if [[ $model_name == \#* ]]; then

  1562.           continue

  1563.         fi

  1564.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1565.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1566.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> "${run_arm64_fp32_log_file}"

  1567.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> adb_run_cmd.txt

  1568.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1569.         if [ $? = 0 ]; then

  1570.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1571.         else

  1572.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1573.         fi

  1574.         # run benchmark test without clib data

  1575.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1576.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1577.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1578.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1579.         if [ $? = 0 ]; then

  1580.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1581.         else

  1582.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1583.         fi

  1584.     done < ${models_tflite_config}

  1585. 

  1586.     # Run caffe converted models:

  1587.     while read line; do

  1588.         model_name=${line}

  1589.         if [[ $model_name == \#* ]]; then

  1590.           continue

  1591.         fi

  1592.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1593.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1594.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> "${run_arm64_fp32_log_file}"

  1595.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> adb_run_cmd.txt

  1596.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1597.         if [ $? = 0 ]; then

  1598.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1599.         else

  1600.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1601.         fi

  1602.         # run benchmark test without clib data

  1603.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1604.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1605.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1606.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1607.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1608.         if [ $? = 0 ]; then

  1609.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1610.         else

  1611.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1612.         fi

  1613.     done < ${models_caffe_config}

  1614. 

  1615.     # Run onnx converted models:

  1616.     while read line; do

  1617.         model_name=${line%;*}

  1618.         length=${#model_name}

  1619.         input_shapes=${line:length+1}

  1620.         if [[ $model_name == \#* ]]; then

  1621.           continue

  1622.         fi

  1623.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1624.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1625.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> "${run_arm64_fp32_log_file}"

  1626.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --inputShapes='${input_shapes}' --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> adb_run_cmd.txt

  1627.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1628.         if [ $? = 0 ]; then

  1629.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1630.         else

  1631.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1632.         fi

  1633.         # run benchmark test without clib data

  1634.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1635.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1636.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=1 --loopCount=2' >> "{run_arm64_fp32_log_file}"

  1637.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1638.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1639.         if [ $? = 0 ]; then

  1640.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1641.         else

  1642.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1643.         fi

  1644.     done < ${models_onnx_config}

  1645. 

  1646.     # Run mindir converted models:

  1647.     while read line; do

  1648.         mindspore_line_info=${line}

  1649.         if [[ $mindspore_line_info == \#* ]]; then

  1650.           continue

  1651.         fi

  1652.         model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'`

  1653.         accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'`

  1654.         echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp32_log_file}"

  1655.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1656.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold='${accuracy_limit} >> "${run_arm64_fp32_log_file}"

  1657.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  1658.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1659.         if [ $? = 0 ]; then

  1660.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1661.         else

  1662.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1663.         fi

  1664.         # run benchmark test without clib data

  1665.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1666.         echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1667.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --warmUpLoopCount=1 --loopCount=2' >> "{run_arm64_fp32_log_file}"

  1668.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1669.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1670.         if [ $? = 0 ]; then

  1671.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1672.         else

  1673.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1674.         fi

  1675.     done < ${models_mindspore_config}

  1676. 

  1677.     # Run mindir converted train models:

  1678.     while read line; do

  1679.         model_name=${line}

  1680.         if [[ $model_name == \#* ]]; then

  1681.           continue

  1682.         fi

  1683.         echo ${model_name}'_train' >> "${run_arm64_fp32_log_file}"

  1684.         echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1685.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.train.ms.out --accuracyThreshold=1.5' >> "${run_arm64_fp32_log_file}"

  1686.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.train.ms.out --accuracyThreshold=1.5' >> adb_run_cmd.txt

  1687.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1688.         if [ $? = 0 ]; then

  1689.             run_result='arm64: '${model_name}'_train pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1690.         else

  1691.             run_result='arm64: '${model_name}'_train failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1692.         fi

  1693.         # run benchmark test without clib data

  1694.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1695.         echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1696.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_train.ms --warmUpLoopCount=1 --loopCount=2' >> "{run_arm64_fp32_log_file}"

  1697.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_train.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1698.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1699.         if [ $? = 0 ]; then

  1700.             run_result='arm64: '${model_name}'_train pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1701.         else

  1702.             run_result='arm64: '${model_name}'_train failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1703.         fi

  1704.     done < ${models_mindspore_train_config}

  1705. 

  1706.     # Run tflite post training quantization converted models:

  1707.     while read line; do

  1708.         posttraining_line_info=${line}

  1709.         if [[ $posttraining_line_info == \#* ]]; then

  1710.           continue

  1711.         fi

  1712.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  1713.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  1714.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1715.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1716.         if [[ $model_name == "transformer_20200831_encoder_fp32.tflite" ]]; then

  1717.             echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --inDataFile=/data/local/tmp/input_output/input/encoder_buffer_in_0-35.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_arm64_fp32_log_file}"

  1718.             echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --inDataFile=/data/local/tmp/input_output/input/encoder_buffer_in_35.bin,/data/local/tmp/input_output/input/encoder_buffer_in_0.bin,/data/local/tmp/input_output/input/encoder_buffer_in_1.bin,/data/local/tmp/input_output/input/encoder_buffer_in_4.bin,/data/local/tmp/input_output/input/encoder_buffer_in_2.bin,/data/local/tmp/input_output/input/encoder_buffer_in_3.bin,/data/local/tmp/input_output/input/encoder_buffer_in_7.bin,/data/local/tmp/input_output/input/encoder_buffer_in_5.bin,/data/local/tmp/input_output/input/encoder_buffer_in_6.bin,/data/local/tmp/input_output/input/encoder_buffer_in_10.bin,/data/local/tmp/input_output/input/encoder_buffer_in_8.bin,/data/local/tmp/input_output/input/encoder_buffer_in_9.bin,/data/local/tmp/input_output/input/encoder_buffer_in_11.bin,/data/local/tmp/input_output/input/encoder_buffer_in_12.bin,/data/local/tmp/input_output/input/encoder_buffer_in_15.bin,/data/local/tmp/input_output/input/encoder_buffer_in_13.bin,/data/local/tmp/input_output/input/encoder_buffer_in_14.bin,/data/local/tmp/input_output/input/encoder_buffer_in_18.bin,/data/local/tmp/input_output/input/encoder_buffer_in_16.bin,/data/local/tmp/input_output/input/encoder_buffer_in_17.bin,/data/local/tmp/input_output/input/encoder_buffer_in_21.bin,/data/local/tmp/input_output/input/encoder_buffer_in_19.bin,/data/local/tmp/input_output/input/encoder_buffer_in_20.bin,/data/local/tmp/input_output/input/encoder_buffer_in_22.bin,/data/local/tmp/input_output/input/encoder_buffer_in_23.bin,/data/local/tmp/input_output/input/encoder_buffer_in_26.bin,/data/local/tmp/input_output/input/encoder_buffer_in_24.bin,/data/local/tmp/input_output/input/encoder_buffer_in_25.bin,/data/local/tmp/input_output/input/encoder_buffer_in_29.bin,/data/local/tmp/input_output/input/encoder_buffer_in_27.bin,/data/local/tmp/input_output/input/encoder_buffer_in_28.bin,/data/local/tmp/input_output/input/encoder_buffer_in_32.bin,/data/local/tmp/input_output/input/encoder_buffer_in_30.bin,/data/local/tmp/input_output/input/encoder_buffer_in_31.bin,/data/local/tmp/input_output/input/encoder_buffer_in_33.bin,/data/local/tmp/input_output/input/encoder_buffer_in_34.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> adb_run_cmd.txt

  1719.         fi

  1720.         if [[ $model_name == "transformer_20200831_decoder_fp32.tflite" ]]; then

  1721.             echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --inDataFile=/data/local/tmp/input_output/input/decoder_buffer_in_0-10.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_arm64_fp32_log_file}"

  1722.             echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --inDataFile=/data/local/tmp/input_output/input/decoder_buffer_in_9.bin,/data/local/tmp/input_output/input/decoder_buffer_in_2.bin,/data/local/tmp/input_output/input/decoder_buffer_in_0.bin,/data/local/tmp/input_output/input/decoder_buffer_in_1.bin,/data/local/tmp/input_output/input/decoder_buffer_in_5.bin,/data/local/tmp/input_output/input/decoder_buffer_in_3.bin,/data/local/tmp/input_output/input/decoder_buffer_in_4.bin,/data/local/tmp/input_output/input/decoder_buffer_in_8.bin,/data/local/tmp/input_output/input/decoder_buffer_in_6.bin,/data/local/tmp/input_output/input/decoder_buffer_in_7.bin,/data/local/tmp/input_output/input/decoder_buffer_in_10.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> adb_run_cmd.txt

  1723.         fi

  1724.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1725.         if [ $? = 0 ]; then

  1726.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1727.         else

  1728.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1729.         fi

  1730.         # run benchmark test without clib data

  1731.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1732.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1733.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1734.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1735.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1736.         if [ $? = 0 ]; then

  1737.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1738.         else

  1739.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1740.         fi

  1741.     done < ${models_tflite_posttraining_config}

  1742. 

  1743.     # Run caffe posttraining models:

  1744.     while read line; do

  1745.         posttraining_line_info=${line}

  1746.         if [[ $posttraining_line_info == \#* ]]; then

  1747.           continue

  1748.         fi

  1749.         model_name=`echo ${posttraining_line_info}|awk -F ' ' '{print $1}'`

  1750.         accuracy_limit=`echo ${posttraining_line_info}|awk -F ' ' '{print $2}'`

  1751.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1752.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1753.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'_posttraining.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> "${run_arm64_fp32_log_file}"

  1754.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'_posttraining.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'_posttraining.ms.out' --accuracyThreshold=${accuracy_limit} >> adb_run_cmd.txt

  1755.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1756.         if [ $? = 0 ]; then

  1757.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1758.         else

  1759.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1760.         fi

  1761.         # run benchmark test without clib data

  1762.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1763.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1764.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1765.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_posttraining.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1766.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1767.         if [ $? = 0 ]; then

  1768.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1769.         else

  1770.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1771.         fi

  1772.     done < ${models_caffe_posttraining_config}

  1773. 

  1774.     # Run tflite aware training quantization converted models:

  1775.     while read line; do

  1776.         model_name=${line}

  1777.         if [[ $model_name == \#* ]]; then

  1778.           continue

  1779.         fi

  1780.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1781.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1782.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> "${run_arm64_fp32_log_file}"

  1783.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> adb_run_cmd.txt

  1784.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1785.         if [ $? = 0 ]; then

  1786.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1787.         else

  1788.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1789.         fi

  1790.         # run benchmark test without clib data

  1791.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1792.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1793.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1794.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1795.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1796.         if [ $? = 0 ]; then

  1797.             run_result='arm64_awq: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1798.         else

  1799.             run_result='arm64_awq: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1800.         fi

  1801.     done < ${models_tflite_awaretraining_config}

  1802. 

  1803.     # Run mindir weightquant converted train models:

  1804.     while read line; do

  1805.         weight_quant_line_info=${line}

  1806.         if [[ $weight_quant_line_info == \#* ]]; then

  1807.           continue

  1808.         fi

  1809.         model_name=`echo ${weight_quant_line_info}|awk -F ' ' '{print $1}'`

  1810.         accuracy_limit=`echo ${weight_quant_line_info}|awk -F ' ' '{print $2}'`

  1811.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1812.         echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1813.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_weightquant.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --loopCount=1 --accuracyThreshold='${accuracy_limit} >> "${run_arm64_fp32_log_file}"

  1814.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'_weightquant.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --loopCount=1 --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  1815.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1816.         if [ $? = 0 ]; then

  1817.             run_result='arm64: '${model_name}'_weightquant pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1818.         else

  1819.             run_result='arm64: '${model_name}'_weightquant failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1820.         fi

  1821.     done < ${models_mindspore_weightquant_config}

  1822. 

  1823.     # Run converted models which has multiple inputs:

  1824.     while read line; do

  1825.         model_name=${line%%;*}

  1826.         if [[ $model_name == \#* ]]; then

  1827.           continue

  1828.         fi

  1829.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  1830.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  1831.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  1832.         input_files=''

  1833.         output_file=''

  1834.         data_path="/data/local/tmp/input_output/"

  1835.         for i in $(seq 1 $input_num)

  1836.         do

  1837.           input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  1838.         done

  1839.         output_file=${data_path}'output/'${model_name}'.ms.out'

  1840.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  1841.           model_name=${model_name%.*}

  1842.         fi

  1843.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1844.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1845.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_arm64_fp32_log_file}"

  1846.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  1847.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1848.         if [ $? = 0 ]; then

  1849.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1850.         else

  1851.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1852.         fi

  1853.         # run benchmark test without clib data

  1854.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1855.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1856.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes} ' --warmUpLoopCount=1 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1857.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes} ' --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1858.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1859.         if [ $? = 0 ]; then

  1860.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1861.         else

  1862.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1863.         fi

  1864.     done < ${models_with_multiple_inputs_config}

  1865. 

  1866.     # Run converted models which does not need to be cared about the accuracy:

  1867.     while read line; do

  1868.         model_name=${line%%;*}

  1869.         if [[ $model_name == \#* ]]; then

  1870.           continue

  1871.         fi

  1872.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  1873.         input_num=`echo ${line} | awk -F ';' '{print $2}'`

  1874.         input_shapes=`echo ${line} | awk -F ';' '{print $3}'`

  1875.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  1876.           model_name=${model_name%.*}

  1877.         fi

  1878.         echo ${model_name} >> "${run_arm64_fp32_log_file}"

  1879.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1880.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=0 --loopCount=2' >> "${run_arm64_fp32_log_file}"

  1881.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=0 --loopCount=2' >> adb_run_cmd.txt

  1882.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp32_log_file}"

  1883.         if [ $? = 0 ]; then

  1884.             run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1885.         else

  1886.             run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1887.         fi

  1888.     done < ${models_for_process_only_config}

  1889. }

  1890. 

  1891. # Run on arm32 platform:

  1892. function Run_arm32() {

  1893.     cd ${arm32_path} || exit 1

  1894.     tar -zxf mindspore-lite-${version}-inference-android-aarch32.tar.gz || exit 1

  1895. 

  1896.     # If build with minddata, copy the minddata related libs

  1897.     cd ${benchmark_test_path} || exit 1

  1898.     if [ -f ${arm32_path}/mindspore-lite-${version}-inference-android-aarch32/inference/lib/libminddata-lite.so ]; then

  1899.         cp -a ${arm32_path}/mindspore-lite-${version}-inference-android-aarch32/inference/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  1900.     fi

  1901. 

  1902.     cp -a ${arm32_path}/mindspore-lite-${version}-inference-android-aarch32/inference/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  1903.     cp -a ${arm32_path}/mindspore-lite-${version}-inference-android-aarch32/tools/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  1904. 

  1905.     # adb push all needed files to the phone

  1906.     adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  1907. 

  1908.     # run adb ,run session ,check the result:

  1909.     echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  1910.     echo 'cp  /data/local/tmp/arm32/libc++_shared.so ./' >> adb_cmd.txt

  1911.     echo 'chmod 777 benchmark' >> adb_cmd.txt

  1912. 

  1913.     adb -s ${device_id} shell < adb_cmd.txt

  1914. 

  1915.     # Run fp32 models:

  1916.     while read line; do

  1917.         model_name=${line}

  1918.         if [[ $model_name == \#* ]]; then

  1919.           continue

  1920.         fi

  1921.         echo ${model_name} >> "${run_arm32_log_file}"

  1922.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1923.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> "${run_arm32_log_file}"

  1924.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out' >> adb_run_cmd.txt

  1925.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm32_log_file}"

  1926.         if [ $? = 0 ]; then

  1927.             run_result='arm32: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1928.         else

  1929.             run_result='arm32: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1930.         fi

  1931.         # run benchmark test without clib data

  1932.         echo ${model_name} >> "${run_arm32_log_file}"

  1933.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1934.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> "${run_arm32_log_file}"

  1935.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --warmUpLoopCount=1 --loopCount=2' >> adb_run_cmd.txt

  1936.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm32_log_file}"

  1937.         if [ $? = 0 ]; then

  1938.             run_result='arm32: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1939.         else

  1940.             run_result='arm32: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1941.         fi

  1942.     done < ${models_arm32_config}

  1943. }

  1944. 

  1945. # Run on arm64-fp16 platform:

  1946. function Run_arm64_fp16() {

  1947.     cd ${arm64_path} || exit 1

  1948.     tar -zxf mindspore-lite-${version}-inference-android-aarch64.tar.gz || exit 1

  1949. 

  1950.     # If build with minddata, copy the minddata related libs

  1951.     cd ${benchmark_test_path} || exit 1

  1952.     if [ -f ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ]; then

  1953.         cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  1954.     fi

  1955.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai.so ${benchmark_test_path}/libhiai.so || exit 1

  1956.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir.so ${benchmark_test_path}/libhiai_ir.so || exit 1

  1957.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir_build.so ${benchmark_test_path}/libhiai_ir_build.so || exit 1

  1958. 

  1959.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  1960.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/tools/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  1961. 

  1962.     # adb push all needed files to the phone

  1963.     adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  1964. 

  1965.     # run adb ,run session ,check the result:

  1966.     echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  1967.     echo 'cp  /data/local/tmp/libc++_shared.so ./' >> adb_cmd.txt

  1968.     echo 'chmod 777 benchmark' >> adb_cmd.txt

  1969. 

  1970.     adb -s ${device_id} shell < adb_cmd.txt

  1971. 

  1972.     # Run fp16 converted models:

  1973.     while read line; do

  1974.         fp16_line_info=${line}

  1975.         if [[ $fp16_line_info == \#* ]]; then

  1976.           continue

  1977.         fi

  1978.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  1979.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`

  1980.         model_name=${model_info%%;*}

  1981.         length=${#model_name}

  1982.         input_shapes=${model_info:length+1}

  1983.         echo "---------------------------------------------------------" >> "${run_arm64_fp16_log_file}"

  1984.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp16_log_file}"

  1985. 

  1986.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  1987.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  1988.         if [[ $accuracy_limit == "-1" ]]; then

  1989.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --enableFp16=true --inputShapes='${input_shapes} >> adb_run_cmd.txt

  1990.         else

  1991.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} ' --inputShapes='${input_shapes} >> adb_run_cmd.txt

  1992.         fi

  1993.         cat adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  1994.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  1995.         if [ $? = 0 ]; then

  1996.             run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  1997.         else

  1998.             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  1999.         fi

  2000.     done < ${models_onnx_fp16_config}

  2001. 

  2002.     while read line; do

  2003.         fp16_line_info=${line}

  2004.         if [[ $fp16_line_info == \#* ]]; then

  2005.           continue

  2006.         fi

  2007.         model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2008.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`

  2009.         echo "---------------------------------------------------------" >> "${run_arm64_fp16_log_file}"

  2010.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp16_log_file}"

  2011. 

  2012.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2013.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2014.         echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2015. 

  2016.         cat adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2017.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2018.         if [ $? = 0 ]; then

  2019.             run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2020.         else

  2021.             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2022.         fi

  2023.     done < ${models_caffe_fp16_config}

  2024. 

  2025.     while read line; do

  2026.         fp16_line_info=${line}

  2027.         if [[ $fp16_line_info == \#* ]]; then

  2028.           continue

  2029.         fi

  2030.         model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2031.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`

  2032.         echo "---------------------------------------------------------" >> "${run_arm64_fp16_log_file}"

  2033.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp16_log_file}"

  2034. 

  2035.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2036.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2037.         echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2038. 

  2039.         cat adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2040.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2041.         if [ $? = 0 ]; then

  2042.             run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2043.         else

  2044.             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2045.         fi

  2046.     done < ${models_tflite_fp16_config}

  2047. 

  2048.     # Run fp16 converted models:

  2049.     while read line; do

  2050.         fp16_line_info=${line}

  2051.         if [[ $fp16_line_info == \#* ]]; then

  2052.           continue

  2053.         fi

  2054.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2055.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`

  2056.         model_name=${model_info%%;*}

  2057.         length=${#model_name}

  2058.         input_shapes=${model_info:length+1}

  2059.         echo "---------------------------------------------------------" >> "${run_arm64_fp16_log_file}"

  2060.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp16_log_file}"

  2061. 

  2062.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2063.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2064.         if [[ $accuracy_limit == "-1" ]]; then

  2065.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --enableFp16=true --inputShapes='${input_shapes} >> adb_run_cmd.txt

  2066.         else

  2067.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} ' --inputShapes='${input_shapes} >> adb_run_cmd.txt

  2068.         fi

  2069.         cat adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2070.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2071.         if [ $? = 0 ]; then

  2072.             run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2073.         else

  2074.             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2075.         fi

  2076.     done < ${models_tf_fp16_config}

  2077. 

  2078.     # Run converted models which has multiple inputs in fp16 mode:

  2079.     while read line; do

  2080.         fp16_line_info=${line}

  2081.         if [[ $fp16_line_info == \#* ]]; then

  2082.           continue

  2083.         fi

  2084.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2085.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`

  2086.         model_name=`echo ${model_info}|awk -F ';' '{print $1}'`

  2087.         input_num=`echo ${model_info} | awk -F ';' '{print $2}'`

  2088.         input_shapes=`echo ${model_info} | awk -F ';' '{print $3}'`

  2089.         input_files=''

  2090.         output_file=''

  2091.         data_path="/data/local/tmp/input_output/"

  2092.         for i in $(seq 1 $input_num)

  2093.         do

  2094.           input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  2095.         done

  2096.         output_file=${data_path}'output/'${model_name}'.ms.out'

  2097.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  2098.           model_name=${model_name%.*}

  2099.         fi

  2100.         echo "---------------------------------------------------------" >> "${run_arm64_fp16_log_file}"

  2101.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_fp16_log_file}"

  2102. 

  2103.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2104.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2105.         echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} '--enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2106. 

  2107.         cat adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2108.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_fp16_log_file}"

  2109.         if [ $? = 0 ]; then

  2110.             run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2111.         else

  2112.             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2113.         fi

  2114.     done < ${models_multiple_inputs_fp16_config}

  2115. }

  2116. 

  2117. # Run on armv8.2-a32-fp16 platform:

  2118. function Run_armv82_a32_fp16() {

  2119.     cd ${armv82_path} || exit 1

  2120.     tar -zxf mindspore-lite-${version}-inference-android-aarch32.tar.gz || exit 1

  2121. 

  2122.     # If build with minddata, copy the minddata related libs

  2123.     cd ${benchmark_test_path} || exit 1

  2124.     if [ -f ${armv82_path}/mindspore-lite-${version}-inference-android-aarch32/inference/minddata/lib/libminddata-lite.so ]; then

  2125.         cp -a ${armv82_path}/mindspore-lite-${version}-inference-android-aarch32/inference/minddata/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  2126.     fi

  2127. 

  2128.     cp -a ${armv82_path}/mindspore-lite-${version}-inference-android-aarch32/inference/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  2129.     cp -a ${armv82_path}/mindspore-lite-${version}-inference-android-aarch32/tools/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  2130. 

  2131.     # adb push all needed files to the phone

  2132.     adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  2133. 

  2134.     # run adb ,run session ,check the result:

  2135.     echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  2136.     echo 'cp  /data/local/tmp/arm32/libc++_shared.so ./' >> adb_cmd.txt

  2137.     echo 'chmod 777 benchmark' >> adb_cmd.txt

  2138. 

  2139.     adb -s ${device_id} shell < adb_cmd.txt

  2140. 

  2141.     # Run fp16 converted models:

  2142.     while read line; do

  2143.         fp16_line_info=${line}

  2144.         column_num=`echo ${fp16_line_info} | awk -F ' ' '{print NF}'`

  2145.         if [[ ${fp16_line_info} == \#* || ${column_num} -lt 3 ]]; then

  2146.           continue

  2147.         fi

  2148.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2149.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $3}'`

  2150.         model_name=${model_info%%;*}

  2151.         length=${#model_name}

  2152.         input_shapes=${model_info:length+1}

  2153.         echo "---------------------------------------------------------" >> "${run_armv82_a32_fp16_log_file}"

  2154.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_armv82_a32_fp16_log_file}"

  2155. 

  2156.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2157.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2158.         if [[ $accuracy_limit == "-1" ]]; then

  2159.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --enableFp16=true --inputShapes='${input_shapes} >> adb_run_cmd.txt

  2160.         else

  2161.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} ' --inputShapes='${input_shapes} >> adb_run_cmd.txt

  2162.         fi

  2163.         cat adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2164.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2165.         if [ $? = 0 ]; then

  2166.             run_result='armv82_a32_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2167.         else

  2168.             run_result='armv82_a32_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2169.         fi

  2170.     done < ${models_onnx_fp16_config}

  2171. 

  2172.     while read line; do

  2173.         fp16_line_info=${line}

  2174.         column_num=`echo ${fp16_line_info} | awk -F ' ' '{print NF}'`

  2175.         if [[ ${fp16_line_info} == \#* || ${column_num} -lt 3 ]]; then

  2176.           continue

  2177.         fi

  2178.         model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2179.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $3}'`

  2180.         echo "---------------------------------------------------------" >> "${run_armv82_a32_fp16_log_file}"

  2181.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_armv82_a32_fp16_log_file}"

  2182. 

  2183.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2184.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2185.         echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2186. 

  2187.         cat adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2188.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2189.         if [ $? = 0 ]; then

  2190.             run_result='armv82_a32_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2191.         else

  2192.             run_result='armv82_a32_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2193.         fi

  2194.     done < ${models_caffe_fp16_config}

  2195. 

  2196.     while read line; do

  2197.         fp16_line_info=${line}

  2198.         column_num=`echo ${fp16_line_info} | awk -F ' ' '{print NF}'`

  2199.         if [[ ${fp16_line_info} == \#* || ${column_num} -lt 3 ]]; then

  2200.           continue

  2201.         fi

  2202.         model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2203.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $3}'`

  2204.         echo "---------------------------------------------------------" >> "${run_armv82_a32_fp16_log_file}"

  2205.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_armv82_a32_fp16_log_file}"

  2206. 

  2207.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2208.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2209.         echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2210. 

  2211.         cat adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2212.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2213.         if [ $? = 0 ]; then

  2214.             run_result='armv82_a32_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2215.         else

  2216.             run_result='armv82_a32_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2217.         fi

  2218.     done < ${models_tflite_fp16_config}

  2219. 

  2220.     # Run fp16 converted models:

  2221.     while read line; do

  2222.         fp16_line_info=${line}

  2223.         column_num=`echo ${fp16_line_info} | awk -F ' ' '{print NF}'`

  2224.         if [[ ${fp16_line_info} == \#* || ${column_num} -lt 3 ]]; then

  2225.           continue

  2226.         fi

  2227.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2228.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $3}'`

  2229.         model_name=${model_info%%;*}

  2230.         length=${#model_name}

  2231.         input_shapes=${model_info:length+1}

  2232.         echo "---------------------------------------------------------" >> "${run_armv82_a32_fp16_log_file}"

  2233.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_armv82_a32_fp16_log_file}"

  2234. 

  2235.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2236.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2237.         if [[ $accuracy_limit == "-1" ]]; then

  2238.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --enableFp16=true --inputShapes='${input_shapes} >> adb_run_cmd.txt

  2239.         else

  2240.           echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} ' --inputShapes='${input_shapes} >> adb_run_cmd.txt

  2241.         fi

  2242.         cat adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2243.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2244.         if [ $? = 0 ]; then

  2245.             run_result='armv82_a32_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2246.         else

  2247.             run_result='armv82_a32_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2248.         fi

  2249.     done < ${models_tf_fp16_config}

  2250. 

  2251.     # Run converted models which has multiple inputs in fp16 mode:

  2252.     while read line; do

  2253.         fp16_line_info=${line}

  2254.         column_num=`echo ${fp16_line_info} | awk -F ' ' '{print NF}'`

  2255.         if [[ ${fp16_line_info} == \#* || ${column_num} -lt 3 ]]; then

  2256.           continue

  2257.         fi

  2258.         model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`

  2259.         accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $3}'`

  2260.         model_name=`echo ${model_info}|awk -F ';' '{print $1}'`

  2261.         input_num=`echo ${model_info} | awk -F ';' '{print $2}'`

  2262.         input_shapes=`echo ${model_info} | awk -F ';' '{print $3}'`

  2263.         input_files=''

  2264.         output_file=''

  2265.         data_path="/data/local/tmp/input_output/"

  2266.         for i in $(seq 1 $input_num)

  2267.         do

  2268.           input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  2269.         done

  2270.         output_file=${data_path}'output/'${model_name}'.ms.out'

  2271.         if [[ ${model_name##*.} == "caffemodel" ]]; then

  2272.           model_name=${model_name%.*}

  2273.         fi

  2274.         echo "---------------------------------------------------------" >> "${run_armv82_a32_fp16_log_file}"

  2275.         echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_armv82_a32_fp16_log_file}"

  2276. 

  2277.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2278.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt

  2279.         echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} '--enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2280. 

  2281.         cat adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2282.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_armv82_a32_fp16_log_file}"

  2283.         if [ $? = 0 ]; then

  2284.             run_result='armv82_a32_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2285.         else

  2286.             run_result='armv82_a32_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2287.         fi

  2288.     done < ${models_multiple_inputs_fp16_config}

  2289. }

  2290. 

  2291. # Run on gpu platform:

  2292. function Run_gpu() {

  2293.     cd ${arm64_path} || exit 1

  2294.     tar -zxf mindspore-lite-${version}-inference-android-aarch64.tar.gz || exit 1

  2295. 

  2296.     # If build with minddata, copy the minddata related libs

  2297.     cd ${benchmark_test_path} || exit 1

  2298.     if [ -f ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ]; then

  2299.         cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  2300.     fi

  2301.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai.so ${benchmark_test_path}/libhiai.so || exit 1

  2302.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir.so ${benchmark_test_path}/libhiai_ir.so || exit 1

  2303.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir_build.so ${benchmark_test_path}/libhiai_ir_build.so || exit 1

  2304. 

  2305.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  2306.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/tools/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  2307. 

  2308.     # adb push all needed files to the phone

  2309.     adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  2310. 

  2311.     # run adb ,run session ,check the result:

  2312.     echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  2313.     echo 'cp  /data/local/tmp/libc++_shared.so ./' >> adb_cmd.txt

  2314.     echo 'chmod 777 benchmark' >> adb_cmd.txt

  2315. 

  2316.     adb -s ${device_id} shell < adb_cmd.txt

  2317. 

  2318.     # Run gpu fp32 converted models:

  2319.     while read line; do

  2320.         model_name=${line%%;*}

  2321.         if [[ $model_name == \#* ]]; then

  2322.           continue

  2323.         fi

  2324.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  2325.         accuracy_limit=`echo ${line} | awk -F ';' '{print $2}'`

  2326.         input_num=`echo ${line} | awk -F ';' '{print $3}'`

  2327.         input_shapes=`echo ${line} | awk -F ';' '{print $4}'`

  2328.         input_files=""

  2329.         data_path="/data/local/tmp/input_output/"

  2330.         output_file=${data_path}'output/'${model_name}'.ms.out'

  2331.         if [[ ${input_num} == "" || ${input_num} == 1 ]]; then

  2332.           input_files=/data/local/tmp/input_output/input/${model_name}.ms.bin

  2333.         else

  2334.           for i in $(seq 1 $input_num)

  2335.           do

  2336.             input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  2337.           done

  2338.         fi

  2339.         if [[ ${accuracy_limit} == "" ]]; then

  2340.           accuracy_limit="0.5"

  2341.         fi

  2342.         echo ${model_name} >> "${run_gpu_log_file}"

  2343.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2344.         if [[ $input_shapes == "" ]]; then

  2345.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> "${run_gpu_log_file}"

  2346.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  2347.         else

  2348.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --inputShapes='${input_shapes}' --accuracyThreshold='${accuracy_limit}' --device=GPU --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> "${run_gpu_log_file}"

  2349.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --inputShapes='${input_shapes}' --accuracyThreshold='${accuracy_limit}' --device=GPU --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  2350.         fi

  2351. 

  2352.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_gpu_log_file}"

  2353.         if [ $? = 0 ]; then

  2354.             run_result='arm64_gpu: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2355.         else

  2356.             run_result='arm64_gpu: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2357.         fi

  2358.     done < ${models_gpu_fp32_config}

  2359. 

  2360.     # Run GPU fp16 converted models:

  2361.     while read line; do

  2362.         model_name=${line}

  2363.         if [[ $model_name == \#* ]]; then

  2364.           continue

  2365.         fi

  2366.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  2367.         accuracy_limit=`echo ${line} | awk -F ';' '{print $2}'`

  2368.         input_num=`echo ${line} | awk -F ';' '{print $3}'`

  2369.         input_shapes=`echo ${line} | awk -F ';' '{print $4}'`

  2370.         input_files=""

  2371.         data_path="/data/local/tmp/input_output/"

  2372.         output_file=${data_path}'output/'${model_name}'.ms.out'

  2373.         if [[ ${input_num} == "" || ${input_num} == 1 ]]; then

  2374.           input_files=/data/local/tmp/input_output/input/${model_name}.ms.bin

  2375.         else

  2376.           for i in $(seq 1 $input_num)

  2377.           do

  2378.             input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  2379.           done

  2380.         fi

  2381.         if [[ ${accuracy_limit} == "" ]]; then

  2382.           accuracy_limit="5"

  2383.         fi

  2384.         echo ${model_name} >> "${run_gpu_log_file}"

  2385.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2386.         if [[ $input_shapes == "" ]]; then

  2387.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> "${run_gpu_log_file}"

  2388.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  2389.         else

  2390.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --inputShapes='${input_shapes}' --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> "${run_gpu_log_file}"

  2391.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --inputShapes='${input_shapes}' --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  2392.         fi

  2393.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_gpu_log_file}"

  2394.         if [ $? = 0 ]; then

  2395.             run_result='arm64_gpu_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2396.         else

  2397.             run_result='arm64_gpu_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2398.         fi

  2399.     done < ${models_gpu_fp16_config}

  2400. 

  2401.     # Run GPU weightquant converted models:

  2402.     while read line; do

  2403.         model_name=${line}

  2404.         if [[ $model_name == \#* ]]; then

  2405.           continue

  2406.         fi

  2407.         model_name=`echo ${line} | awk -F ';' '{print $1}'`

  2408.         accuracy_limit=`echo ${line} | awk -F ';' '{print $2}'`

  2409.         input_num=`echo ${line} | awk -F ';' '{print $3}'`

  2410.         input_shapes=`echo ${line} | awk -F ';' '{print $4}'`

  2411.         input_files=""

  2412.         data_path="/data/local/tmp/input_output/"

  2413.         output_file=${data_path}'output/'${model_name}'.ms.out'

  2414.         if [[ ${input_num} == "" || ${input_num} == 1 ]]; then

  2415.           input_files=/data/local/tmp/input_output/input/${model_name}.ms.bin

  2416.         else

  2417.           for i in $(seq 1 $input_num)

  2418.           do

  2419.             input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  2420.           done

  2421.         fi

  2422.         if [[ ${accuracy_limit} == "" ]]; then

  2423.           accuracy_limit="5"

  2424.         fi

  2425.         echo ${model_name} >> "${run_gpu_log_file}"

  2426.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2427.         if [[ $input_shapes == "" ]]; then

  2428.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> "${run_gpu_log_file}"

  2429.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  2430.         else

  2431.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --inputShapes='${input_shapes}' --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> "${run_gpu_log_file}"

  2432.           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=GPU --enableFp16=true --inputShapes='${input_shapes}' --accuracyThreshold='${accuracy_limit}' --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile='${output_file} >> adb_run_cmd.txt

  2433.         fi

  2434.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_gpu_log_file}"

  2435.         if [ $? = 0 ]; then

  2436.             run_result='arm64_gpu_weightquant: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2437.         else

  2438.             run_result='arm64_gpu_weightquant: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2439.         fi

  2440.     done < ${models_gpu_weightquant_config}

  2441. }

  2442. 

  2443. # Run on npu platform:

  2444. function Run_npu() {

  2445.     cd ${arm64_path} || exit 1

  2446.     tar -zxf mindspore-lite-${version}-inference-android-aarch64.tar.gz || exit 1

  2447. 

  2448.     # If build with minddata, copy the minddata related libs

  2449.     cd ${benchmark_test_path} || exit 1

  2450.     if [ -f ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ]; then

  2451.         cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libminddata-lite.so ${benchmark_test_path}/libminddata-lite.so || exit 1

  2452.     fi

  2453.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai.so ${benchmark_test_path}/libhiai.so || exit 1

  2454.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir.so ${benchmark_test_path}/libhiai_ir.so || exit 1

  2455.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/third_party/hiai_ddk/lib/libhiai_ir_build.so ${benchmark_test_path}/libhiai_ir_build.so || exit 1

  2456. 

  2457.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/inference/lib/libmindspore-lite.so ${benchmark_test_path}/libmindspore-lite.so || exit 1

  2458.     cp -a ${arm64_path}/mindspore-lite-${version}-inference-android-aarch64/tools/benchmark/benchmark ${benchmark_test_path}/benchmark || exit 1

  2459. 

  2460.     # adb push all needed files to the phone

  2461.     adb -s ${device_id} push ${benchmark_test_path} /data/local/tmp/ > adb_push_log.txt

  2462. 

  2463.     # run adb ,run session ,check the result:

  2464.     echo 'cd  /data/local/tmp/benchmark_test' > adb_cmd.txt

  2465.     echo 'cp  /data/local/tmp/libc++_shared.so ./' >> adb_cmd.txt

  2466.     echo 'chmod 777 benchmark' >> adb_cmd.txt

  2467. 

  2468.     adb -s ${device_id} shell < adb_cmd.txt

  2469. 

  2470.     # Run npu converted models:

  2471.     while read line; do

  2472.         model_line_info=${line}

  2473.         if [[ $model_line_info == \#* ]]; then

  2474.           continue

  2475.         fi

  2476.         model_name=`echo ${line}|awk -F ' ' '{print $1}'`

  2477.         accuracy_limit=`echo ${line}|awk -F ' ' '{print $2}'`

  2478.         input_num=`echo ${line}|awk -F ' ' '{print $3}'`

  2479.         data_path="/data/local/tmp/input_output/"

  2480.         input_files=''

  2481.         if [[ -z "$input_num" || $input_num == 1 ]]; then

  2482.           input_files=${data_path}'input/'$model_name'.ms.bin'

  2483.         elif [[ ! -z "$input_num" && $input_num -gt 1 ]]; then

  2484.           for i in $(seq 1 $input_num)

  2485.           do

  2486.             input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i','

  2487.           done

  2488.         fi

  2489.         echo "mindspore run npu: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_npu_log_file}"

  2490.         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt

  2491.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=NPU --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold='${accuracy_limit} >> "${run_npu_log_file}"

  2492.         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --device=NPU --modelFile='${model_name}'.ms --inDataFile='${input_files}' --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt

  2493.         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_npu_log_file}"

  2494.         if [ $? = 0 ]; then

  2495.             run_result='arm64_npu: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2496.         else

  2497.             run_result='arm64_npu: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2498.         fi

  2499.     done < ${models_npu_config}

  2500. }

  2501. 

  2502. # Run on x86 java platform:

  2503. function Run_x86_java() {

  2504.     cd ${x86_java_path} || exit 1

  2505.     tar -zxf mindspore-lite-${version}-inference-linux-x64-jar.tar.gz || exit 1

  2506.     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${x86_java_path}/mindspore-lite-${version}-inference-linux-x64-jar/jar

  2507.     # compile benchmark

  2508.     echo "javac -cp ${x86_java_path}/mindspore-lite-${version}-inference-linux-x64-jar/jar/mindspore-lite-java.jar ${basepath}/st/java/src/main/java/Benchmark.java -d ."

  2509.     javac -cp ${x86_java_path}/mindspore-lite-${version}-inference-linux-x64-jar/jar/mindspore-lite-java.jar ${basepath}/st/java/src/main/java/Benchmark.java -d .

  2510. 

  2511.     count=0

  2512.     # Run tflite converted models:

  2513.     while read line; do

  2514.         # only run top5.

  2515.         count=`expr ${count}+1`

  2516.         if [[ ${count} -gt 5 ]]; then

  2517.             break

  2518.         fi

  2519.         model_name=${line}

  2520.         if [[ $model_name == \#* ]]; then

  2521.           continue

  2522.         fi

  2523.         echo ${model_name} >> "${run_x86_java_log_file}"

  2524.         echo "java -classpath .:${x86_java_path}/mindspore-lite-${version}-inference-linux-x64-jar/jar/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms '${models_path}'/input_output/input/${model_name}.ms.bin '${models_path}'/input_output/output/${model_name}.ms.out 1" >> "${run_x86_java_log_file}"

  2525.         java -classpath .:${x86_java_path}/mindspore-lite-${version}-inference-linux-x64-jar/jar/mindspore-lite-java.jar Benchmark ${ms_models_path}/${model_name}.ms ${models_path}/input_output/input/${model_name}.ms.bin ${models_path}/input_output/output/${model_name}.ms.out 1

  2526.         if [ $? = 0 ]; then

  2527.             run_result='x86_java: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2528.         else

  2529.             run_result='x86_java: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2530.         fi

  2531.     done < ${models_tflite_config}

  2532. }

  2533. 

  2534. # Run on x86 codegen benchmark

  2535. function Run_x86_codegen() {

  2536.     local CODEGEN_PATH=${x86_path}/mindspore-lite-${version}-inference-linux-x64/tools/codegen

  2537. 

  2538.     rm -rf ${build_path}

  2539.     mkdir -p ${build_path}

  2540. 

  2541.     while read line; do

  2542.         model_name=${line}

  2543.         if [[ $model_name == \#* ]]; then

  2544.           continue

  2545.         fi

  2546.         echo ${model_name} >> "${run_x86_codegen_log_file}"

  2547.         local PARALLEL=$1

  2548.         ${CODEGEN_PATH}/codegen --codePath=${build_path} --modelPath=${ms_models_path}/${model_name}.ms --supportParallel=${PARALLEL} >> ${run_x86_codegen_log_file}

  2549.         # 1. build benchmark

  2550.         mkdir -p ${build_path}/${model_name}/build && cd ${build_path}/${model_name}/build || exit 1

  2551.         cmake -DPKG_PATH=${x86_path}/mindspore-lite-${version}-inference-linux-x64 ${build_path}/${model_name} >> ${run_x86_codegen_log_file}

  2552.         make >> ${run_x86_codegen_log_file}

  2553.         # 2. run benchmark

  2554.         echo "net file: ${build_path}/${model_name}/src/net.bin" >> ${run_x86_codegen_log_file}

  2555.         if [[ "${PARALLEL}" == "false" ]]; then

  2556.           echo "./benchmark ${models_path}/input_output/input/${model_name}.ms.bin ${build_path}/${model_name}/src/net.bin 1 ${models_path}/input_output/output/${model_name}.ms.out" >> ${run_x86_codegen_log_file}

  2557.           ./benchmark ${models_path}/input_output/input/${model_name}.ms.bin ${build_path}/${model_name}/src/net.bin 1 ${models_path}/input_output/output/${model_name}.ms.out >> ${run_x86_codegen_log_file}

  2558.         else

  2559.           echo "./benchmark ${models_path}/input_output/input/${model_name}.ms.bin ${build_path}/${model_name}/src/net.bin 1 ${models_path}/input_output/output/${model_name}.ms.out 4 0" >> ${run_x86_codegen_log_file}

  2560.           ./benchmark ${models_path}/input_output/input/${model_name}.ms.bin ${build_path}/${model_name}/src/net.bin 1 ${models_path}/input_output/output/${model_name}.ms.out 4 0 >> ${run_x86_codegen_log_file}

  2561.         fi

  2562.         if [ $? = 0 ]; then

  2563.             run_result='x86_codegen: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}

  2564.         else

  2565.             run_result='x86_codegen: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1

  2566.         fi

  2567.     done < ${models_codegen_config}

  2568. 

  2569.     rm -rf ${build_path}

  2570. }

  2571. 

  2572. # Print start msg before run testcase

  2573. function MS_PRINT_TESTCASE_START_MSG() {

  2574.     echo ""

  2575.     echo -e "-----------------------------------------------------------------------------------------------------------------------------------"

  2576.     echo -e "env                  Testcase                                                                                             Result   "

  2577.     echo -e "---                  --------                                                                                             ------   "

  2578. }

  2579. 

  2580. # Print start msg after run testcase

  2581. function MS_PRINT_TESTCASE_END_MSG() {

  2582.     echo -e "-----------------------------------------------------------------------------------------------------------------------------------"

  2583. }

  2584. 

  2585. function Print_Converter_Result() {

  2586.     MS_PRINT_TESTCASE_END_MSG

  2587.     while read line; do

  2588.         arr=("${line}")

  2589.         printf "%-15s %-20s %-90s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]} ${arr[3]}

  2590.     done < ${run_converter_result_file}

  2591.     MS_PRINT_TESTCASE_END_MSG

  2592. }

  2593. 

  2594. function Print_Benchmark_Result() {

  2595.     MS_PRINT_TESTCASE_START_MSG

  2596.     while read line; do

  2597.         arr=("${line}")

  2598.         printf "%-20s %-100s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]}

  2599.     done < ${run_benchmark_result_file}

  2600.     MS_PRINT_TESTCASE_END_MSG

  2601. }

  2602. 

  2603. basepath=$(pwd)

  2604. echo ${basepath}

  2605. #set -e

  2606. 

  2607. # Example:sh run_benchmark_nets.sh -r /home/temp_test -m /home/temp_test/models -d "8KE5T19620002408" -e arm_cpu

  2608. while getopts "r:m:d:e:" opt; do

  2609.     case ${opt} in

  2610.         r)

  2611.             release_path=${OPTARG}

  2612.             echo "release_path is ${OPTARG}"

  2613.             ;;

  2614.         m)

  2615.             models_path=${OPTARG}

  2616.             echo "models_path is ${OPTARG}"

  2617.             ;;

  2618.         d)

  2619.             device_id=${OPTARG}

  2620.             echo "device_id is ${OPTARG}"

  2621.             ;;

  2622.         e)

  2623.             backend=${OPTARG}

  2624.             echo "backend is ${OPTARG}"

  2625.             ;;

  2626.         ?)

  2627.         echo "unknown para"

  2628.         exit 1;;

  2629.     esac

  2630. done

  2631. 

  2632. # mkdir train

  2633. 

  2634. x86_path=${release_path}/ubuntu_x86

  2635. file_name=$(ls ${x86_path}/*inference-linux-x64.tar.gz)

  2636. IFS="-" read -r -a file_name_array <<< "$file_name"

  2637. version=${file_name_array[2]}

  2638. 

  2639. # Set models config filepath

  2640. models_tflite_config=${basepath}/models_tflite.cfg

  2641. models_tf_config=${basepath}/models_tf.cfg

  2642. models_caffe_config=${basepath}/models_caffe.cfg

  2643. models_caffe_weightquant_config=${basepath}/models_caffe_weightquant.cfg

  2644. models_tflite_awaretraining_config=${basepath}/models_tflite_awaretraining.cfg

  2645. models_tflite_posttraining_config=${basepath}/models_tflite_posttraining.cfg

  2646. models_caffe_posttraining_config=${basepath}/models_caffe_posttraining.cfg

  2647. models_tflite_weightquant_config=${basepath}/models_tflite_weightquant.cfg

  2648. models_onnx_config=${basepath}/models_onnx.cfg

  2649. models_onnx_fp16_config=${basepath}/models_onnx_fp16.cfg

  2650. models_caffe_fp16_config=${basepath}/models_caffe_fp16.cfg

  2651. models_tflite_fp16_config=${basepath}/models_tflite_fp16.cfg

  2652. models_tf_fp16_config=${basepath}/models_tf_fp16.cfg

  2653. models_multiple_inputs_fp16_config=${basepath}/models_with_multiple_inputs_fp16.cfg

  2654. models_mindspore_config=${basepath}/models_mindspore.cfg

  2655. models_mindspore_train_config=${basepath}/models_mindspore_train.cfg

  2656. models_mindspore_mixbit_config=${basepath}/models_mindspore_mixbit.cfg

  2657. models_gpu_fp32_config=${basepath}/models_gpu_fp32.cfg

  2658. models_gpu_fp16_config=${basepath}/models_gpu_fp16.cfg

  2659. models_gpu_weightquant_config=${basepath}/models_gpu_weightquant.cfg

  2660. models_mindspore_weightquant_config=${basepath}/models_mindspore_weightquant.cfg

  2661. models_arm32_config=${basepath}/models_arm32.cfg

  2662. models_npu_config=${basepath}/models_npu.cfg

  2663. models_compatibility_config=${basepath}/models_compatibility.cfg

  2664. models_with_multiple_inputs_config=${basepath}/models_with_multiple_inputs.cfg

  2665. models_for_process_only_config=${basepath}/models_for_process_only.cfg

  2666. models_tf_weightquant_config=${basepath}/models_tf_weightquant.cfg

  2667. models_codegen_config=${basepath}/codegen/models_codegen.cfg

  2668. 

  2669. ms_models_path=${basepath}/ms_models

  2670. build_path=${basepath}/codegen_build

  2671. 

  2672. # Write converter result to temp file

  2673. run_converter_log_file=${basepath}/run_converter_log.txt

  2674. echo ' ' > ${run_converter_log_file}

  2675. 

  2676. run_converter_result_file=${basepath}/run_converter_result.txt

  2677. echo ' ' > ${run_converter_result_file}

  2678. 

  2679. # Run converter

  2680. echo "start Run converter ..."

  2681. Run_Converter

  2682. Run_converter_PID=$!

  2683. sleep 1

  2684. 

  2685. wait ${Run_converter_PID}

  2686. Run_converter_status=$?

  2687. 

  2688. # Check converter result and return value

  2689. if [[ ${Run_converter_status} = 0 ]];then

  2690.     echo "Run converter success"

  2691.     Print_Converter_Result

  2692. else

  2693.     echo "Run converter failed"

  2694.     cat ${run_converter_log_file}

  2695.     Print_Converter_Result

  2696.     exit 1

  2697. fi

  2698. 

  2699. # Write benchmark result to temp file

  2700. run_benchmark_result_file=${basepath}/run_benchmark_result.txt

  2701. echo ' ' > ${run_benchmark_result_file}

  2702. 

  2703. run_x86_log_file=${basepath}/run_x86_log.txt

  2704. echo 'run x86 logs: ' > ${run_x86_log_file}

  2705. 

  2706. run_x86_sse_log_file=${basepath}/run_x86_sse_log.txt

  2707. echo 'run x86 sse logs: ' > ${run_x86_sse_log_file}

  2708. 

  2709. run_x86_avx_log_file=${basepath}/run_x86_avx_log.txt

  2710. echo 'run x86 avx logs: ' > ${run_x86_avx_log_file}

  2711. 

  2712. run_x86_java_log_file=${basepath}/run_x86_java_log.txt

  2713. echo 'run x86 java logs: ' > ${run_x86_java_log_file}

  2714. 

  2715. run_x86_codegen_log_file=${basepath}/run_x86_codegen_log.txt

  2716. echo 'run x86 codegen logs: ' > ${run_x86_codegen_log_file}

  2717. 

  2718. run_arm64_fp32_log_file=${basepath}/run_arm64_fp32_log.txt

  2719. echo 'run arm64_fp32 logs: ' > ${run_arm64_fp32_log_file}

  2720. 

  2721. run_arm64_fp32_codegen_log_file=${basepath}/run_arm64_fp32_codegen_log.txt

  2722. echo 'run arm64_codegen logs: ' > ${run_arm64_fp32_codegen_log_file}

  2723. 

  2724. run_arm32_fp32_codegen_log_file=${basepath}/run_arm32_fp32_codegen_log.txt

  2725. echo 'run arm32_codegen logs: ' > ${run_arm32_fp32_codegen_log_file}

  2726. 

  2727. run_arm64_fp16_log_file=${basepath}/run_arm64_fp16_log.txt

  2728. echo 'run arm64_fp16 logs: ' > ${run_arm64_fp16_log_file}

  2729. 

  2730. run_armv82_a32_fp16_log_file=${basepath}/run_armv82_a32_fp16_log.txt

  2731. echo 'run arm82_a32_fp16 logs: ' > ${run_armv82_a32_fp16_log_file}

  2732. 

  2733. run_arm32_log_file=${basepath}/run_arm32_log.txt

  2734. echo 'run arm32 logs: ' > ${run_arm32_log_file}

  2735. 

  2736. run_gpu_log_file=${basepath}/run_gpu_log.txt

  2737. echo 'run gpu logs: ' > ${run_gpu_log_file}

  2738. 

  2739. run_npu_log_file=${basepath}/run_npu_log.txt

  2740. echo 'run npu logs: ' > ${run_npu_log_file}

  2741. 

  2742. # Copy the MindSpore models:

  2743. echo "Push files to the arm and run benchmark"

  2744. benchmark_test_path=${basepath}/benchmark_test

  2745. rm -rf ${benchmark_test_path}

  2746. mkdir -p ${benchmark_test_path}

  2747. cp -a ${ms_models_path}/*.ms ${benchmark_test_path} || exit 1

  2748. # Copy models converted using old release of mslite converter for compatibility test

  2749. cp -a ${models_path}/compatibility_test/*.ms ${benchmark_test_path} || exit 1

  2750. 

  2751. backend=${backend:-"all"}

  2752. isFailed=0

  2753. 

  2754. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86" ]]; then

  2755.     # Run on x86

  2756.     echo "start Run x86 ..."

  2757.     Run_x86 &

  2758.     Run_x86_PID=$!

  2759.     sleep 1

  2760. fi

  2761. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-sse" ]]; then

  2762.     # Run on x86-sse

  2763.     echo "start Run x86 sse ..."

  2764.     Run_x86_sse &

  2765.     Run_x86_sse_PID=$!

  2766.     sleep 1

  2767. fi

  2768. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-avx" ]]; then

  2769.     # Run on x86-avx

  2770.     echo "start Run x86 avx ..."

  2771.     Run_x86_avx &

  2772.     Run_x86_avx_PID=$!

  2773.     sleep 1

  2774. fi

  2775. 

  2776. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-java" ]]; then

  2777.     # Run on x86-java

  2778.     echo "start Run x86 java ..."

  2779.     x86_java_path=${release_path}/aar/avx

  2780.     Run_x86_java &

  2781.     Run_x86_java_PID=$!

  2782.     sleep 1

  2783. fi

  2784. 

  2785. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-codegen" ]]; then

  2786.     # Run on x86-java

  2787.     echo "start Run x86 codegen ..."

  2788.     Run_x86_codegen "false" &

  2789.     Run_x86_codegen_PID=$!

  2790.     sleep 1

  2791. fi

  2792. 

  2793. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_codegen" ]]; then

  2794.     # Run on arm64

  2795.     arm64_path=${release_path}/android_aarch64

  2796.     file_name=$(ls ${arm64_path}/*inference-android-aarch64.tar.gz)

  2797.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2798.     version=${file_name_array[2]}

  2799. 

  2800.     echo "start Run arm64 codegen ..."

  2801.     Run_arm64_codegen

  2802.     Run_arm64_codegen_status=$?

  2803.     sleep 1

  2804. fi

  2805. 

  2806. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm32_codegen" ]]; then

  2807.     # Run on arm32

  2808.     arm32_path=${release_path}/android_aarch32

  2809.     file_name=$(ls ${arm32_path}/*inference-android-aarch32.tar.gz)

  2810.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2811.     version=${file_name_array[2]}

  2812. 

  2813.     echo "start Run arm32 codegen ..."

  2814.     Run_arm32_codegen

  2815.     Run_arm32_codegen_status=$?

  2816.     sleep 1

  2817. fi

  2818. 

  2819. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm32_fp16" ]]; then

  2820.     # Run on armv82-a32-fp16

  2821.     armv82_path=${release_path}/android_aarch32

  2822.     file_name=$(ls ${armv82_path}/*inference-android-aarch32.tar.gz)

  2823.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2824.     version=${file_name_array[2]}

  2825. 

  2826.     echo "start Run armv82-a32-fp16 ..."

  2827.     Run_armv82_a32_fp16

  2828.     Run_armv82_a32_fp16_status=$?

  2829.     sleep 1

  2830. fi

  2831. 

  2832. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm32_fp32" ]]; then

  2833.     # Run on arm32

  2834.     arm32_path=${release_path}/android_aarch32

  2835.     # mv ${arm32_path}/*train-android-aarch32* ./train

  2836.     file_name=$(ls ${arm32_path}/*inference-android-aarch32.tar.gz)

  2837.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2838.     version=${file_name_array[2]}

  2839. 

  2840.     echo "start Run arm32 ..."

  2841.     Run_arm32

  2842.     Run_arm32_status=$?

  2843.     sleep 1

  2844. fi

  2845. 

  2846. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_fp32" ]]; then

  2847.     # Run on arm64

  2848.     arm64_path=${release_path}/android_aarch64

  2849.     # mv ${arm64_path}/*train-android-aarch64* ./train

  2850.     file_name=$(ls ${arm64_path}/*inference-android-aarch64.tar.gz)

  2851.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2852.     version=${file_name_array[2]}

  2853. 

  2854.     echo "start Run arm64 ..."

  2855.     Run_arm64

  2856.     Run_arm64_fp32_status=$?

  2857.     sleep 1

  2858. fi

  2859. 

  2860. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_fp16" ]]; then

  2861.     # Run on arm64-fp16

  2862.     arm64_path=${release_path}/android_aarch64

  2863.     # mv ${arm64_path}/*train-android-aarch64* ./train

  2864.     file_name=$(ls ${arm64_path}/*inference-android-aarch64.tar.gz)

  2865.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2866.     version=${file_name_array[2]}

  2867. 

  2868.     echo "start Run arm64-fp16 ..."

  2869.     Run_arm64_fp16

  2870.     Run_arm64_fp16_status=$?

  2871.     sleep 1

  2872. fi

  2873. 

  2874. if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "gpu" ]]; then

  2875.     # Run on gpu

  2876.     arm64_path=${release_path}/android_aarch64

  2877.     # mv ${arm64_path}/*train-android-aarch64* ./train

  2878.     file_name=$(ls ${arm64_path}/*inference-android-aarch64.tar.gz)

  2879.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2880.     version=${file_name_array[2]}

  2881. 

  2882.     echo "start Run gpu ..."

  2883.     Run_gpu

  2884.     Run_gpu_status=$?

  2885.     sleep 1

  2886. fi

  2887. if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "npu" ]]; then

  2888.     # Run on npu

  2889.     arm64_path=${release_path}/android_aarch64

  2890.     # mv ${arm64_path}/*train-android-aarch64* ./train

  2891.     file_name=$(ls ${arm64_path}/*inference-android-aarch64.tar.gz)

  2892.     IFS="-" read -r -a file_name_array <<< "$file_name"

  2893.     version=${file_name_array[2]}

  2894. 

  2895.     echo "start Run npu ..."

  2896.     Run_npu

  2897.     Run_npu_status=$?

  2898.     sleep 1

  2899. fi

  2900. 

  2901. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-sse" ]]; then

  2902.     wait ${Run_x86_sse_PID}

  2903.     Run_x86_sse_status=$?

  2904. 

  2905.     if [[ ${Run_x86_sse_status} != 0 ]];then

  2906.         echo "Run_x86 sse failed"

  2907.         cat ${run_x86_sse_log_file}

  2908.         isFailed=1

  2909.     fi

  2910. fi

  2911. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-avx" ]]; then

  2912.     wait ${Run_x86_avx_PID}

  2913.     Run_x86_avx_status=$?

  2914. 

  2915.     if [[ ${Run_x86_avx_status} != 0 ]];then

  2916.         echo "Run_x86 avx failed"

  2917.         cat ${run_x86_avx_log_file}

  2918.         isFailed=1

  2919.     fi

  2920. fi

  2921. 

  2922. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86" ]]; then

  2923.     wait ${Run_x86_PID}

  2924.     Run_x86_status=$?

  2925. 

  2926.     # Check benchmark result and return value

  2927.     if [[ ${Run_x86_status} != 0 ]];then

  2928.         echo "Run_x86 failed"

  2929.         cat ${run_x86_log_file}

  2930.         isFailed=1

  2931.     fi

  2932. fi

  2933. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-java" ]]; then

  2934.     wait ${Run_x86_java_PID}

  2935.     Run_x86_java_status=$?

  2936. 

  2937.     if [[ ${Run_x86_java_status} != 0 ]];then

  2938.         echo "Run_x86 java failed"

  2939.         cat ${run_x86_java_log_file}

  2940.         isFailed=1

  2941.     fi

  2942. fi

  2943. 

  2944. if [[ $backend == "all" || $backend == "x86-all" || $backend == "x86-codegen" ]]; then

  2945.     wait ${Run_x86_codegen_PID}

  2946.     Run_x86_codegen_status=$?

  2947. 

  2948.     if [[ ${Run_x86_codegen_status} != 0 ]];then

  2949.         echo "Run_x86 codegen failed"

  2950.         cat ${run_x86_codegen_log_file}

  2951.         isFailed=1

  2952.     fi

  2953. fi

  2954. 

  2955. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_fp32" ]]; then

  2956.     if [[ ${Run_arm64_fp32_status} != 0 ]];then

  2957.         echo "Run_arm64_fp32 failed"

  2958.         cat ${run_arm64_fp32_log_file}

  2959.         isFailed=1

  2960.     fi

  2961. fi

  2962. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_codegen" ]]; then

  2963.     if [[ ${Run_arm64_codegen_status} != 0 ]];then

  2964.         echo "Run_arm64_codegen failed"

  2965.         cat ${run_arm64_fp32_codegen_log_file}

  2966.         isFailed=1

  2967.     fi

  2968. fi

  2969. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm32_codegen" ]]; then

  2970.     if [[ ${Run_arm32_codegen_status} != 0 ]];then

  2971.         echo "Run_arm32 codegen failed"

  2972.         cat ${run_arm32_fp32_codegen_log_file}

  2973.         isFailed=1

  2974.     fi

  2975. fi

  2976. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_fp16" ]]; then

  2977.     if [[ ${Run_arm64_fp16_status} != 0 ]];then

  2978.         echo "Run_arm64_fp16 failed"

  2979.         cat ${run_arm64_fp16_log_file}

  2980.         isFailed=1

  2981.     fi

  2982. fi

  2983. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm32_fp16" ]]; then

  2984.     if [[ ${Run_armv82_a32_fp16_status} != 0 ]];then

  2985.         echo "Run_armv82_a32_fp16 failed"

  2986.         cat ${run_armv82_a32_fp16_log_file}

  2987.         isFailed=1

  2988.     fi

  2989. fi

  2990. if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm32_fp32" ]]; then

  2991.     if [[ ${Run_arm32_status} != 0 ]];then

  2992.         echo "Run_arm32 failed"

  2993.         cat ${run_arm32_log_file}

  2994.         isFailed=1

  2995.     fi

  2996. fi

  2997. if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "gpu" ]]; then

  2998.     if [[ ${Run_gpu_status} != 0 ]];then

  2999.         echo "Run_gpu failed"

  3000.         cat ${run_gpu_log_file}

  3001.         isFailed=1

  3002.     fi

  3003. fi

  3004. if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "npu" ]]; then

  3005.     if [[ ${Run_npu_status} != 0 ]];then

  3006.         echo "Run_npu failed"

  3007.         cat ${run_npu_log_file}

  3008.         isFailed=1

  3009.     fi

  3010. fi

  3011. 

  3012. if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "npu" ]]; then

  3013.     cd ${basepath} || exit 1

  3014.     bash ${basepath}/run_cropper.sh -r ${release_path} -d ${device_id}

  3015.     Run_cropper_status=$?

  3016.     if [[ ${Run_cropper_status} != 0 ]];then

  3017.         echo "Run cropper failed"

  3018.         cat ${run_npu_log_file}

  3019.         isFailed=1

  3020.         exit 1

  3021.     fi

  3022. fi

  3023. 

  3024. echo "Run_x86 and Run_x86_sse and Run_x86_avx and Run_arm64_fp32 and Run_arm64_fp16 and Run_arm32_fp32 and Run_armv82_a32_fp16 and Run_gpu and Run_npu and  is ended"

  3025. Print_Benchmark_Result

  3026. if [[ $isFailed == 1 ]]; then

  3027.     exit 1

  3028. fi

  3029. exit 0