diff --git a/mindspore/lite/test/models_mindspore.cfg b/mindspore/lite/test/models_mindspore.cfg index 3b8a8a12b1..e2d0510c1e 100644 --- a/mindspore/lite/test/models_mindspore.cfg +++ b/mindspore/lite/test/models_mindspore.cfg @@ -1,12 +1,13 @@ -ssd.mindir -mobilenetv2_438.mindir -gate_u_net_small-1_110.mindir -shufflenetv2.mindir -inceptionv3.mindir -googlenet.mindir -retinaface_732_1280_iod.mindir -mobilefacenet_iod.mindir -effnet_iod.mindir -resnext50.mindir -ocr_mobilenetV2.mindir -mindspore_ghostnet_ssd_13x.mindir +ssd.mindir 1.5 +mobilenetv2_438.mindir 1.5 +gate_u_net_small-1_110.mindir 1.5 +shufflenetv2.mindir 1.5 +inceptionv3.mindir 1.5 +googlenet.mindir 1.5 +retinaface_732_1280_iod.mindir 1.5 +mobilefacenet_iod.mindir 1.5 +effnet_iod.mindir 1.5 +resnext50.mindir 1.5 +ocr_mobilenetV2.mindir 1.5 +mobilenet_quant.mindir 5 +mindspore_ghostnet_ssd_13x.mindir 1.5 diff --git a/mindspore/lite/test/run_benchmark_nets.sh b/mindspore/lite/test/run_benchmark_nets.sh index 8509a7624c..f3603926e0 100644 --- a/mindspore/lite/test/run_benchmark_nets.sh +++ b/mindspore/lite/test/run_benchmark_nets.sh @@ -68,10 +68,12 @@ function Run_Converter() { # Convert mindspore models: while read line; do - model_name=${line} - if [[ $model_name == \#* ]]; then + mindspore_line_info=${line} + if [[ $mindspore_line_info == \#* ]]; then continue fi + model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'` echo ${model_name} >> "${run_converter_log_file}" echo './converter_lite --fmk=MINDIR --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name}'' >> "${run_converter_log_file}" ./converter_lite --fmk=MINDIR --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name} @@ -385,15 +387,18 @@ function Run_x86() { # Run mindspore converted models: while read line; do - model_name=${line} - if [[ $model_name == \#* ]]; then + mindspore_line_info=${line} + if [[ $mindspore_line_info == \#* ]]; then continue fi - echo ${model_name} >> "${run_x86_log_file}" + model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'` + echo "---------------------------------------------------------" >> "${run_x86_log_file}" + echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_x86_log_file}" echo 'cd '${x86_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86} >> "${run_x86_log_file}" cd ${x86_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86} || return 1 echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out' >> "${run_x86_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --benchmarkDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --accuracyThreshold=1.5 >> "${run_x86_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --benchmarkDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} else @@ -633,15 +638,18 @@ function Run_x86_sse() { # Run mindspore converted models: while read line; do - model_name=${line} - if [[ $model_name == \#* ]]; then + mindspore_line_info=${line} + if [[ $mindspore_line_info == \#* ]]; then continue fi - echo ${model_name} >> "${run_x86_sse_log_file}" + model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'` + echo "---------------------------------------------------------" >> "${run_x86_sse_log_file}" + echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_x86_sse_log_file}" echo 'cd '${x86_path}'/mindspore-lite-'${version}'-runtime-x86-sse-'${process_unit_x86} >> "${run_x86_sse_log_file}" cd ${x86_path}/mindspore-lite-${version}-runtime-x86-sse-${process_unit_x86} || return 1 echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/'${model_name}'.ms.out' >> "${run_x86_sse_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --benchmarkDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --accuracyThreshold=1.5 >> "${run_x86_sse_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/input/${model_name}.ms.bin --benchmarkDataFile=/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/output/${model_name}.ms.out --accuracyThreshold=${accuracy_limit} >> "${run_x86_sse_log_file}" if [ $? = 0 ]; then run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} else @@ -1047,14 +1055,16 @@ function Run_arm64() { # Run mindir converted models: while read line; do - model_name=${line} - if [[ $model_name == \#* ]]; then + mindspore_line_info=${line} + if [[ $mindspore_line_info == \#* ]]; then continue fi - echo ${model_name} >> "${run_arm64_log_file}" + model_name=`echo ${mindspore_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${mindspore_line_info}|awk -F ' ' '{print $2}'` + echo "mindspore run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_log_file}" echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold=1.5' >> "${run_arm64_log_file}" - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold=1.5' >> adb_run_cmd.txt + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold='${accuracy_limit} >> "${run_arm64_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}" if [ $? = 0 ]; then run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}