diff --git a/mindspore/lite/test/models_with_several_inputs_or_without_outputs.cfg b/mindspore/lite/test/models_for_process_only.cfg similarity index 67% rename from mindspore/lite/test/models_with_several_inputs_or_without_outputs.cfg rename to mindspore/lite/test/models_for_process_only.cfg index 2d1251765a..0eed3361c9 100644 --- a/mindspore/lite/test/models_with_several_inputs_or_without_outputs.cfg +++ b/mindspore/lite/test/models_for_process_only.cfg @@ -1,7 +1,6 @@ lite-model_arbitrary-image-stylization-inceptionv3_dr_transfer_1.tflite lite-model_arbitrary-image-stylization-inceptionv3_int8_transfer_1.tflite -lite-model_arbitrary-image-stylization-inceptionv3_fp16_transfer_1.tflite;2 -# lite-model_arbitrary-image-stylization-inceptionv3-dynamic-shapes_dr_transfer_1.tflite # has nan input for rsqrt +lite-model_arbitrary-image-stylization-inceptionv3-dynamic-shapes_dr_transfer_1.tflite;2;1,1,1,100:1,64,64,3 lite-model_cartoongan_dr_1.tflite mindspore_efficientnet_b0.mindir mindspore_efficientnet_b4minus.mindir @@ -14,10 +13,8 @@ lite-model_deeplabv3-mobilenetv2-ade20k_1_default_1.tflite lite-model_deeplabv3-mobilenetv2_dm05-int8_1_default_1.tflite lite-model_deeplabv3-mobilenetv3-cityscapes_1_default_1.tflite lite-model_east-text-detector_dr_1.tflite -magenta_arbitrary-image-stylization-v1-256_fp16_transfer_1.tflite;2 magenta_arbitrary-image-stylization-v1-256_int8_transfer_1.tflite magenta_arbitrary-image-stylization-v1-256_int8_prediction_1.tflite -# albert_lite_base_squadv1_1.tflite;3 # input need in range [0,2) efficientnet_lite0_int8_2.tflite efficientnet_lite1_int8_2.tflite efficientnet_lite2_int8_2.tflite @@ -26,14 +23,8 @@ efficientnet_lite4_int8_2.tflite mtk_transformer_encoder.tflite mtk_transformer_decoder_joint.tflite ml_ei_facedetection.onnx -# mobilebert_1_default_1.tflite;3 # input need in range [0,2) quant_aware_bank_card_detection_inception.onnx quant_aware_bank_card_recognition_fcny.onnx quant_aware_identify_card_detect.onnx -porseg_tmp.onnx;2 -ml_video_edit_img_segment_adaptise.pb;2 -ml_video_edit_img_segment_adaptise_pb2tflite.tflite;2 -ml_video_edit_video_segment_gauss_adaptis_part2.pb;2 -ml_video_edit_video_segment_gauss_adaptis_part2_pb2tflite.tflite;2 tiny-yolov3-11.onnx;2;1,416,416,3:1,2 ml_video_edit_person_divison_pic;2 diff --git a/mindspore/lite/test/models_onnx_fp16.cfg b/mindspore/lite/test/models_onnx_fp16.cfg index c71a920f40..5f568f46ed 100644 --- a/mindspore/lite/test/models_onnx_fp16.cfg +++ b/mindspore/lite/test/models_onnx_fp16.cfg @@ -22,10 +22,10 @@ residual_distill_bs_32.onnx 30 ml_face_3d.onnx 2 gts_version-RFB-320_simplified.onnx 3 mnist-8.onnx 10 -#crnn_lite_lstm_v2.onnx 0.1 -#psenet_lite_mbv2.onnx 0.1 -#super-resolution-10.onnx 1 -#tinyyolov2-8.onnx 0.3 +crnn_lite_lstm_v2.onnx;32,32,32,1 0.3 +psenet_lite_mbv2.onnx;1,32,32,3 0.6 +super-resolution-10.onnx;1,224,224,1 4.5 +tinyyolov2-8.onnx;1,416,416,3 5.5 ml_2012_ocr_cn.onnx 200 #ml_2012_ocr_cn_noLSTM.onnx 1 candy-9.onnx 5 diff --git a/mindspore/lite/test/models_with_multiple_inputs.cfg b/mindspore/lite/test/models_with_multiple_inputs.cfg new file mode 100644 index 0000000000..8fa2410fd6 --- /dev/null +++ b/mindspore/lite/test/models_with_multiple_inputs.cfg @@ -0,0 +1,10 @@ +lite-model_arbitrary-image-stylization-inceptionv3_fp16_transfer_1.tflite;2 +magenta_arbitrary-image-stylization-v1-256_fp16_transfer_1.tflite;2 +# albert_lite_base_squadv1_1.tflite;3 +# mobilebert_1_default_1.tflite;3 +porseg_tmp.onnx;2 +ml_video_edit_img_segment_adaptise.pb;2 +ml_video_edit_img_segment_adaptise_pb2tflite.tflite;2 +ml_video_edit_video_segment_gauss_adaptis_part2.pb;2 +ml_video_edit_video_segment_gauss_adaptis_part2_pb2tflite.tflite;2 +decoder.onnx;2;1,7,512:1,7 diff --git a/mindspore/lite/test/run_benchmark_nets.sh b/mindspore/lite/test/run_benchmark_nets.sh index 739c4732a7..27421bcf45 100755 --- a/mindspore/lite/test/run_benchmark_nets.sh +++ b/mindspore/lite/test/run_benchmark_nets.sh @@ -179,7 +179,8 @@ function Run_Converter() { if [[ $fp16_line_info == \#* ]]; then continue fi - model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + model_name=${model_info%%;*} echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms' cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms if [ $? = 0 ]; then @@ -279,7 +280,42 @@ function Run_Converter() { fi done < ${models_mindspore_mixbit_config} - # Convert models which has several inputs or does not need to be cared about the accuracy: + # Convert models which has multiple inputs: + while read line; do + if [[ $line == \#* ]]; then + continue + fi + model_name=${line%%;*} + model_type=${model_name##*.} + case $model_type in + pb) + model_fmk="TF" + ;; + tflite) + model_fmk="TFLITE" + ;; + caffemodel) + model_name=${model_name%.*} + model_fmk="CAFFE" + ;; + onnx) + model_fmk="ONNX" + ;; + mindir) + model_fmk="MINDIR" + ;; + esac + echo ${model_name} >> "${run_converter_log_file}" + echo './converter_lite --fmk='${model_fmk}' --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}" + ./converter_lite --fmk=${model_fmk} --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name} + if [ $? = 0 ]; then + converter_result='converter '${model_type}' '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file} + else + converter_result='converter '${model_type}' '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1 + fi + done < ${models_with_multiple_inputs_config} + + # Convert models which does not need to be cared about the accuracy: while read line; do if [[ $line == \#* ]]; then continue @@ -318,7 +354,7 @@ function Run_Converter() { else converter_result='converter '${model_type}' '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1 fi - done < ${models_only_for_process_config} + done < ${models_for_process_only_config} } # Run on x86 platform: @@ -592,7 +628,7 @@ function Run_x86() { fi done < ${models_mindspore_mixbit_config} - # Run converted models which has several inputs or does not need to be cared about the accuracy: + # Run converted models which has multiple inputs: while read line; do if [[ $line == \#* ]]; then continue @@ -603,31 +639,51 @@ function Run_x86() { input_files='' output_file='' data_path="/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/" - if [[ -z "$input_num" || $input_num == 1 ]] && [ -e ${data_path}'input/'${model_name}'.ms.bin' ]; then - input_files=${data_path}'input/'$model_name'.ms.bin' - elif [[ ! -z "$input_num" && $input_num -gt 1 ]]; then - for i in $(seq 1 $input_num) - do - input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' - done + for i in $(seq 1 $input_num) + do + input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' + done + output_file=${data_path}'output/'${model_name}'.ms.out' + if [[ ${model_name##*.} == "caffemodel" ]]; then + model_name=${model_name%.*} fi - if [ -e ${data_path}'output/'${model_name}'.ms.out' ]; then - output_file=${data_path}'output/'${model_name}'.ms.out' + echo ${model_name} >> "${run_x86_log_file}" + echo 'cd '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64' >> "{run_x86_log_file}" + cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64 || return 1 + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_x86_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} >> "${run_x86_log_file}" + if [ $? = 0 ]; then + run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} + else + run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi + done < ${models_with_multiple_inputs_config} + + # Run converted models which does not need to be cared about the accuracy: + while read line; do + if [[ $line == \#* ]]; then + continue + fi + model_name=`echo ${line} | awk -F ';' '{print $1}'` + input_num=`echo ${line} | awk -F ';' '{print $2}'` + input_shapes=`echo ${line} | awk -F ';' '{print $3}'` + input_files='' + output_file='' + data_path="/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/" if [[ ${model_name##*.} == "caffemodel" ]]; then model_name=${model_name%.*} fi echo ${model_name} >> "${run_x86_log_file}" echo 'cd '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64' >> "{run_x86_log_file}" cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64 || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file}' --loopCount=1 --warmUpLoopCount=0' >> "${run_x86_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} --loopCount=1 --warmUpLoopCount=0 >> "${run_x86_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inputShapes='${input_shapes}' --loopCount=2 --warmUpLoopCount=0' >> "${run_x86_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inputShapes=${input_shapes} --loopCount=2 --warmUpLoopCount=0 >> "${run_x86_log_file}" if [ $? = 0 ]; then run_result='x86: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} else run_result='x86: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi - done < ${models_only_for_process_config} + done < ${models_for_process_only_config} } # Run on x86 sse platform: @@ -865,7 +921,7 @@ function Run_x86_sse() { fi done < ${models_mindspore_mixbit_config} - # Run converted models which has several inputs or does not need to be cared about the accuracy: + # Run converted models with multiple inputs: while read line; do model_name=${line%%;*} if [[ $model_name == \#* ]]; then @@ -877,31 +933,49 @@ function Run_x86_sse() { input_files='' output_file='' data_path="/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/" - if [[ -z "$input_num" || $input_num == 1 ]] && [ -e ${data_path}'input/'${model_name}'.ms.bin' ]; then - input_files=${data_path}'input/'$model_name'.ms.bin' - elif [[ ! -z "$input_num" && $input_num -gt 1 ]]; then - for i in $(seq 1 $input_num) - do - input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' - done + for i in $(seq 1 $input_num) + do + input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' + done + output_file=${data_path}'output/'${model_name}'.ms.out' + if [[ ${model_name##*.} == "caffemodel" ]]; then + model_name=${model_name%.*} fi - if [ -e ${data_path}'output/'${model_name}'.ms.out' ]; then - output_file=${data_path}'output/'${model_name}'.ms.out' + echo ${model_name} >> "${run_x86_sse_log_file}" + echo 'cd '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64-sse' >> "{run_x86_sse_log_file}" + cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64-sse || return 1 + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_x86_sse_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} >> "${run_x86_sse_log_file}" + if [ $? = 0 ]; then + run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} + else + run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 + fi + done < ${models_with_multiple_inputs_config} + + # Run converted models which does not need to be cared about the accuracy: + while read line; do + model_name=${line%%;*} + if [[ $model_name == \#* ]]; then + continue fi + model_name=`echo ${line} | awk -F ';' '{print $1}'` + input_num=`echo ${line} | awk -F ';' '{print $2}'` + input_shapes=`echo ${line} | awk -F ';' '{print $3}'` if [[ ${model_name##*.} == "caffemodel" ]]; then model_name=${model_name%.*} fi echo ${model_name} >> "${run_x86_sse_log_file}" echo 'cd '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64-sse' >> "{run_x86_sse_log_file}" cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64-sse || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file}' --loopCount=1 --warmUpLoopCount=0' >> "${run_x86_sse_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} --loopCount=1 --warmUpLoopCount=0 >> "${run_x86_sse_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inputShapes='${input_shapes}' --loopCount=2 --warmUpLoopCount=0' >> "${run_x86_sse_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inputShapes=${input_shapes} --loopCount=2 --warmUpLoopCount=0 >> "${run_x86_sse_log_file}" if [ $? = 0 ]; then run_result='x86_sse: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} else run_result='x86_sse: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi - done < ${models_only_for_process_config} + done < ${models_for_process_only_config} } # Run on x86 avx platform: @@ -1139,7 +1213,7 @@ function Run_x86_avx() { fi done < ${models_mindspore_mixbit_config} - # Run converted models which has several inputs or does not need to be cared about the accuracy: + # Run converted models which has multiple inputs: while read line; do model_name=${line%%;*} if [[ $model_name == \#* ]]; then @@ -1151,31 +1225,49 @@ function Run_x86_avx() { input_files='' output_file='' data_path="/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/" - if [[ -z "$input_num" || $input_num == 1 ]] && [ -e ${data_path}'input/'${model_name}'.ms.bin' ]; then - input_files=${data_path}'input/'$model_name'.ms.bin' - elif [[ ! -z "$input_num" && $input_num -gt 1 ]]; then - for i in $(seq 1 $input_num) - do - input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' - done + for i in $(seq 1 $input_num) + do + input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' + done + output_file=${data_path}'output/'${model_name}'.ms.out' + if [[ ${model_name##*.} == "caffemodel" ]]; then + model_name=${model_name%.*} + fi + echo ${model_name} >> "${run_x86_avx_log_file}" + echo 'cd '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64-avx' >> "{run_x86_avx_log_file}" + cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64-avx || return 1 + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file} >> "${run_x86_avx_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} >> "${run_x86_avx_log_file}" + if [ $? = 0 ]; then + run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} + else + run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi - if [ -e ${data_path}'output/'${model_name}'.ms.out' ]; then - output_file=${data_path}'output/'${model_name}'.ms.out' + done < ${models_with_multiple_inputs_config} + + # Run converted models which does not need to be cared about the accuracy: + while read line; do + model_name=${line%%;*} + if [[ $model_name == \#* ]]; then + continue fi + model_name=`echo ${line} | awk -F ';' '{print $1}'` + input_num=`echo ${line} | awk -F ';' '{print $2}'` + input_shapes=`echo ${line} | awk -F ';' '{print $3}'` if [[ ${model_name##*.} == "caffemodel" ]]; then model_name=${model_name%.*} fi echo ${model_name} >> "${run_x86_avx_log_file}" echo 'cd '${x86_path}'/mindspore-lite-'${version}'-inference-linux-x64-avx' >> "{run_x86_avx_log_file}" cd ${x86_path}/mindspore-lite-${version}-inference-linux-x64-avx || return 1 - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inDataFile='${input_files}' --inputShapes='${input_shapes}' --benchmarkDataFile='${output_file}' --loopCount=1 --warmUpLoopCount=0' >> "${run_x86_avx_log_file}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inDataFile=${input_files} --inputShapes=${input_shapes} --benchmarkDataFile=${output_file} --loopCount=1 --warmUpLoopCount=0 >> "${run_x86_avx_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile='${ms_models_path}'/'${model_name}'.ms --inputShapes='${input_shapes}' --loopCount=2 --warmUpLoopCount=0' >> "${run_x86_avx_log_file}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./benchmark/benchmark --modelFile=${ms_models_path}/${model_name}.ms --inputShapes=${input_shapes} --loopCount=2 --warmUpLoopCount=0 >> "${run_x86_avx_log_file}" if [ $? = 0 ]; then run_result='x86_avx: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} else run_result='x86_avx: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi - done < ${models_only_for_process_config} + done < ${models_for_process_only_config} } # Run on arm64 platform: @@ -1439,14 +1531,17 @@ function Run_arm64() { if [[ $fp16_line_info == \#* ]]; then continue fi - model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + model_info=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'` + model_name=${model_info%%;*} + length=${#model_name} + input_shapes=${model_info:length+1} echo "---------------------------------------------------------" >> "${run_arm64_log_file}" echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_log_file}" echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt - echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt + echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} ' --inputShapes='${input_shapes} >> adb_run_cmd.txt cat adb_run_cmd.txt >> "${run_arm64_log_file}" adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}" @@ -1695,7 +1790,7 @@ function Run_arm64() { fi done < ${models_npu_config} - # Run converted models which has several inputs or does not need to be cared about the accuracy: + # Run converted models which has multiple inputs: while read line; do model_name=${line%%;*} if [[ $model_name == \#* ]]; then @@ -1707,18 +1802,11 @@ function Run_arm64() { input_files='' output_file='' data_path="/data/local/tmp/input_output/" - model_x86_path="/home/workspace/mindspore_dataset/mslite/models/hiai/input_output/" - if [[ -z "$input_num" || $input_num == 1 ]] && [ -e ${model_x86_path}'input/'${model_name}'.ms.bin' ]; then - input_files=${data_path}'input/'$model_name'.ms.bin' - elif [[ ! -z "$input_num" && $input_num -gt 1 ]]; then - for i in $(seq 1 $input_num) - do - input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' - done - fi - if [ -e ${model_x86_path}'output/'${model_name}'.ms.out' ]; then - output_file=${data_path}'output/'${model_name}'.ms.out' - fi + for i in $(seq 1 $input_num) + do + input_files=$input_files${data_path}'input/'$model_name'.ms.bin_'$i',' + done + output_file=${data_path}'output/'${model_name}'.ms.out' if [[ ${model_name##*.} == "caffemodel" ]]; then model_name=${model_name%.*} fi @@ -1732,7 +1820,31 @@ function Run_arm64() { else run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi - done < ${models_only_for_process_config} + done < ${models_with_multiple_inputs_config} + + # Run converted models which does not need to be cared about the accuracy: + while read line; do + model_name=${line%%;*} + if [[ $model_name == \#* ]]; then + continue + fi + model_name=`echo ${line} | awk -F ';' '{print $1}'` + input_num=`echo ${line} | awk -F ';' '{print $2}'` + input_shapes=`echo ${line} | awk -F ';' '{print $3}'` + if [[ ${model_name##*.} == "caffemodel" ]]; then + model_name=${model_name%.*} + fi + echo ${model_name} >> "${run_arm64_log_file}" + echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=0 --loopCount=2' >> "${run_arm64_log_file}" + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test;./benchmark --modelFile='${model_name}'.ms --inputShapes='${input_shapes}' --warmUpLoopCount=0 --loopCount=2' >> adb_run_cmd.txt + adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}" + if [ $? = 0 ]; then + run_result='arm64: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} + else + run_result='arm64: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 + fi + done < ${models_for_process_only_config} } # Run on arm32 platform: @@ -1876,7 +1988,8 @@ models_mindspore_weightquant_config=${basepath}/models_mindspore_weightquant.cfg models_arm32_config=${basepath}/models_arm32.cfg models_npu_config=${basepath}/models_npu.cfg models_compatibility_config=${basepath}/models_compatibility.cfg -models_only_for_process_config=${basepath}/models_with_several_inputs_or_without_outputs.cfg +models_with_multiple_inputs_config=${basepath}/models_with_multiple_inputs.cfg +models_for_process_only_config=${basepath}/models_for_process_only.cfg ms_models_path=${basepath}/ms_models