diff --git a/mindspore/lite/test/models_caffe_fp16.cfg b/mindspore/lite/test/models_caffe_fp16.cfg new file mode 100644 index 0000000000..85c058e68e --- /dev/null +++ b/mindspore/lite/test/models_caffe_fp16.cfg @@ -0,0 +1,68 @@ +age_medium 6 +beard 2 +emotion 60 +gender_res_large_deploy 0.1 +glasses 4 +hat 1 +isface 1 +ml_bank_detect_0312 20 +ml_face_div_parsing 8 +ml_hardware_eyeclose 0.1 +ml_ocr_detect_20200305 10 +Mnet6_0312_extract_pay 15 +pose_3d 90 +RFB-Epoch-170-no-transpose 4 +tracking 4 +mtk_landmark 1 +mtk_pose_tuku 1 +mtk_face_recognition_v1 20 +mtk_2012_ATLANTA_10class_20190614_v41 4 +mtk_detect-deeper-halfdeeper-mbv1-lastearlySSD-shortcut-400-400_nopostprocess_simplified 4 +detect-deeper-halfdeeper-mbv1-shortcut-400-400_nopostprocess_simplified 1 +hiai_face_detect_rfb 4 +hiai_face_isface 0.1 +hiai_face_landmark 0.2 +hiai_face_pose_tuku 1.3 +ml_hand_detection 8 +ml_ocr_cn 6 +ml_ocr_sfz_detect_0325 3 +ml_hardware_liveness 3 +ml_liveness_detect_landmark 1 +ml_face_contour 0.5 +2012_ATLANTA_1class_20190621_v4.x_nomean 1 +ml_ocr_sfz_add_final_0325 0.1 +ml_hardware_pose 2 +ml_bank_recog 0.1 +2012_ATLANTA_10class_20190131_v4.0 12 +mnet 9 +recognition 10 +ml_face_landmark 1 +model_hebing_3branch 40 +hiai_cv_focusShootOCRModel_07 3 +hiai_cv_focusShootOCRModel_03 60 +hiai_cv_focusShootOCRModel_01 14 +hiai_face_hat1 1 +hiai_cv_focusShootOCRModel_04 8 +hiai_cv_focusShootOCRModel_06 13 +hiai_cpu_face_hat 0.3 +hiai_video_seg 1 +hiai_semantic_seg 3 +hiai_human_seg 28 +hiai_face_recognition_1 10 +hiai_cpu_face_detect 4 +hiai_cpu_face_attr 12 +hiai_face_attr1 12 +detect-mbv1-shortcut-400-400_nopostprocess_simplified 8 +detect_mbv1_640_480_nopostprocess_simplified 6 +retinaface 6 +deconv_test_model 20 +deconvs_model 1 +HWSR-s_256_256 10 +age_new 22 +detection_retinaface_fix 13 +landmark 1 +plat_isface 6 +PoseNet_dla_17_x512 5 +ml_location_scene_division 8 +ml_tabel_recog 0.1 +ml_text_division 12 diff --git a/mindspore/lite/test/models_fp16.cfg b/mindspore/lite/test/models_fp16.cfg deleted file mode 100644 index c3efbf0705..0000000000 --- a/mindspore/lite/test/models_fp16.cfg +++ /dev/null @@ -1,34 +0,0 @@ -hiai_cpu_face_detect 4 -#hiai_cpu_face_attr 5000 -hiai_cpu_face_hat 0.3 -hiai_face_detect_rfb 4 -hiai_face_isface 0.1 -hiai_face_landmark 0.2 -hiai_face_pose_tuku 1.3 -ml_face_contour 0.5 -ml_face_landmark 1 -mtk_isface 0.2 -mtk_landmark 0.3 -mtk_pose_tuku 1 -mtk_age_gender.tflite 0.1 -mtk_model_face_dress.tflite 1 -mtk_model_ckpt.tflite 20 -mtk_face_features_v1.tflite 20 -mtk_new_detect.tflite 3 -mtk_pose.tflite 2 -mtk_model_emotions_0727_nosoftmax.tflite 2 -#mtk_model_normalize_object_scene_ps_20200519_f32.tflite 20 -mtk_model_normalize_object_scene_ps_20200826_f32_no_softmax.tflite 22 -mtk_276landmark_0913.tflite 16 -mtk_face_recognition.tflite 8 -mtk_convert_model.tflite 5 -mobilenet_v1_0.5_128.tflite 2.5 -mobilenet_v1_1.0_192.tflite 26 -mobilenet_v2_1.0_224.tflite 3 -retinaface 6 -deconvs_model 1 -efficientnet_lite4_fp32_2.tflite 6 -mnasnet_0.50_224_1_metadata_1.tflite 5 -detect-deeper-halfdeeper-mbv1-shortcut-400-400_nopostprocess_simplified 1 -HWSR-s_256_256 10 -#model_name accuracy_limit diff --git a/mindspore/lite/test/models_onnx_fp16.cfg b/mindspore/lite/test/models_onnx_fp16.cfg new file mode 100644 index 0000000000..dec09e4812 --- /dev/null +++ b/mindspore/lite/test/models_onnx_fp16.cfg @@ -0,0 +1,34 @@ +mtk_detect-mbv2-shortcut-400-400-simplified.onnx 4 +mtk_emotions-d2012-75.8%.onnx 20 +mtk_face_features_v3.onnx 20 +emotion-ferplus-8.onnx 1 +#rcnn-ilsvrc13-9.onnx 0.1 +efficientnet-lite4-11.onnx 2 +mobilenetv2-7.onnx 8 +shufflenet-v2-10.onnx 5 +squeezenet1.1-7.onnx 1 +densenet-9.onnx 6 +googlenet-9.onnx 3 +inception-v1-9.onnx 3 +inception-v2-9.onnx 4 +shufflenet-9.onnx 1000 +squeezenet1.0-9.onnx 5 +residual_distill_cifar10_bs_1.onnx 1 +residual_distill_cifar10_bs_32.onnx 2 +residual_distill_bs_1.onnx 6 +residual_distill_bs_32.onnx 30 +ml_face_3d.onnx 2 +gts_version-RFB-320_simplified.onnx 3 +mnist-8.onnx 10 +#crnn_lite_lstm_v2.onnx 0.1 +#psenet_lite_mbv2.onnx 0.1 +#super-resolution-10.onnx 1 +#tinyyolov2-8.onnx 0.3 +ml_2012_ocr_cn.onnx 200 +#ml_2012_ocr_cn_noLSTM.onnx 0.1 +candy-9.onnx 5 +mosaic-9.onnx 4 +pointilism-9.onnx 3 +rain-princess-9.onnx 5 +udnie-9.onnx 3 +adversarial_pruning.onnx 3 diff --git a/mindspore/lite/test/models_tflite_fp16.cfg b/mindspore/lite/test/models_tflite_fp16.cfg new file mode 100644 index 0000000000..f4740a7bbf --- /dev/null +++ b/mindspore/lite/test/models_tflite_fp16.cfg @@ -0,0 +1,134 @@ +hiai_model_0909_kd_rot_ps_softmax.tflite 10 +hiai_chinese_english_recognize_model_float32.tflite 13 +hiai_bigmodel_ghost_2_1_no_normalized_no_trans_tflite.tflite 10 +hiai_bigmodel_ghost_5_1_no_normalized_no_trans_tflite.tflite 10 +hiai_cn_recognize_modify_padv2.tflite 14 +hiai_model_normalize_object_scene_ps_20200519.tflite 20 +#hiai_detectmodel_06_23_960_480_1180700.tflite 20 +#hiai_detect_curve_model_float32.tflite 20 +#hiai_detectmodel_desnet_256_128_64_32.tflite 20 +mtk_AADB_HADB_MBV2_model_fp32.tflite 5 +mtk_AADB_HADB_MBV3_model_fp32.tflite 6 +mobilenet_v1_0.25_128.tflite 5 +mobilenet_v1_0.25_160.tflite 5 +mobilenet_v1_0.25_192.tflite 3 +mobilenet_v1_0.25_224.tflite 3 +mobilenet_v1_0.5_128.tflite 2.5 +mobilenet_v1_0.5_160.tflite 5 +mobilenet_v1_0.5_192.tflite 6 +mobilenet_v1_0.5_224.tflite 5 +mobilenet_v1_0.75_128.tflite 6 +mobilenet_v1_0.75_160.tflite 6 +mobilenet_v1_0.75_192.tflite 6 +mobilenet_v1_0.75_224.tflite 6 +mobilenet_v1_1.0_128.tflite 10 +mobilenet_v1_1.0_160.tflite 6 +mobilenet_v1_1.0_192.tflite 26 +mobilenet_v1_1.0_224.tflite 6 +mobilenet_v2_1.0_224.tflite 3 +mtk_model_normalize_object_scene_ps_20200519_f32.tflite 20 +mtk_model_ckpt.tflite 20 +mtk_age_gender.tflite 0.1 +mtk_model_face_dress.tflite 2 +mtk_face_features_v1.tflite 20 +densenet.tflite 36 +squeezenet.tflite 6 +resnet_v2_101_299.tflite 3 +mnasnet_1.3_224.tflite 12 +inception_v3.tflite 3 +deeplabv3_257_mv_gpu.tflite 3 +multi_person_mobilenet_v1_075_float.tflite 9 +#hiai_vad.tflite 20 +ide_label_base.tflite 22 +ide_label_retrained.tflite 80 +ml_ei_headpose.tflite 3 +ml_ei_landmark.tflite 3 +mnist.tflite 4 +mobilenet.tflite 0.1 +resnet.tflite 120 +scan_hms_angle1.tflite 4 +scan_hms_detect.tflite 12 +hiai_latin_ocr.tflite 45 +hiai_latin_ocr_1.tflite 13 +ml_ocr_jk.tflite 2 +nasnet_mobile.tflite 3 +nasnet_large.tflite 3 +inception_resnet_v2.tflite 10 +ml_ocr_latin.tflite 15 +hiai_PoseEstimation_Pcm.tflite 15 +hiai_ssd_mobilenetv2_object.tflite 60 +hiai_cv_focusShootOCRModel_02.tflite 13 +hiai_cv_poseEstimation.tflite 190 +inception_v4.tflite 10 +mtk_model_normalize_object_scene_ps_20200519_f16.tflite 10 +mtk_model_face_dress_fp16.tflite 45 +mtk_AADB_HADB_MBV2_model_f16.tflite 5 +mtk_AADB_HADB_MBV3_model_f16.tflite 10 +mtk_model_emotions_0725_fp16.tflite 3 +mtk_face_features_v1_fp16.tflite 20 +siteAI_digcom_AI_ECN.tflite 0.1 +siteAI_digcom_g2v_keras.tflite 5 +siteAI_trans_nonlinear.tflite 0.1 +siteAI_trans_tcpclassify.tflite 5 +siteAI_wireless_depress_w.tflite 8 +siteAI_wireless_restore_w.tflite 0.1 +magenta_arbitrary-image-stylization-v1-256_fp16_prediction_1.tflite 5 +ml_object_detect.tflite 3 +ml_object_detect_1.tflite 5 +hiai_cpu_face_emotion.tflite 6 +hiai_cpu_face_gazing.tflite 5 +hiai_cpu_face_headpose.tflite 100 +hiai_humanDetection.tflite 15 +#hiai_cv_focusShootOCRModel_08.tflite 0.3 +#ml_face_openclose.tflite 40 +hiai_face_model_npu.tflite 5 +hiai_ctpn_feature_map.tflite 10 +hiai_cv_labelDetectorModel_v2.tflite 30 +#hiai_cv_labelDetectorModel_v3.tflite 20 +hiai_cv_labelDetectorModel_v4.tflite 3 +hiai_dress_detect.tflite 3 +hiai_cv_saliencyDetectorModel.tflite 3 +hiai_frozen_inference_graph.tflite 5 +hiai_ghostnet.tflite 3 +#hiai_iMaxDN_RGB.tflite 0.4 +#hiai_iMaxSR_RGB.tflite 1 +hiai_label_and_video.tflite 10 +hiai_lm_inference_graph.tflite 3 +efficientnet_lite0_fp32_2.tflite 6 +efficientnet_lite1_fp32_2.tflite 6 +efficientnet_lite2_fp32_2.tflite 3 +efficientnet_lite3_fp32_2.tflite 6 +efficientnet_lite4_fp32_2.tflite 6 +mnasnet_0.50_224_1_metadata_1.tflite 5 +mnasnet_0.75_224_1_metadata_1.tflite 6 +mnasnet_1.0_128_1_metadata_1.tflite 6 +mnasnet_1.0_160_1_metadata_1.tflite 6 +mnasnet_1.0_192_1_metadata_1.tflite 8 +mnasnet_1.0_224_1_metadata_1.tflite 6 +mnasnet_1.0_96_1_metadata_1.tflite 6 +# ml_vision_guide_detection1.tflite 20 +# ml_vision_guide_detection3.tflite 20 +lite-model_on_device_vision_classifier_popular_us_products_V1_1.tflite 16 +lite-model_on_device_vision_classifier_popular_wine_V1_1.tflite 80 +posenet_mobilenet_float_075_1_default_1.tflite 45 +deeplabv3_1_default_1.tflite 6 +lite-model_deeplabv3-mobilenetv2_dm05-float16_1_default_1.tflite 13 +lite-model_deeplabv3-mobilenetv2-float16_1_default_1.tflite 60 +lite-model_east-text-detector_fp16_1.tflite 60 +lite-model_cartoongan_fp16_1.tflite 3 +lite-model_arbitrary-image-stylization-inceptionv3_fp16_predict_1.tflite 6 +gts_detect_5k_tf115.tflite 6 +mtk_isface.tflite 0.2 +mtk_landmark.tflite 0.1 +mtk_new_detect.tflite 3 +mtk_pose.tflite 2 +mtk_model_emotions_0727_nosoftmax.tflite 2 +mtk_model_normalize_object_scene_ps_20200826_f32_no_softmax.tflite 22 +mtk_276landmark_0913.tflite 16 +mtk_face_recognition.tflite 8 +mtk_convert_model.tflite 5 +smartreply.tflite 0.1 +mindspore_text_classification_tflite.tflite 4 +#ml_location.tflite 0.1 +ml_text_correction.tflite 1 +#ml_pic_shopping.tflite 0.1 diff --git a/mindspore/lite/test/run_benchmark_nets.sh b/mindspore/lite/test/run_benchmark_nets.sh index f3603926e0..127631c9ba 100644 --- a/mindspore/lite/test/run_benchmark_nets.sh +++ b/mindspore/lite/test/run_benchmark_nets.sh @@ -164,7 +164,37 @@ function Run_Converter() { else converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1 fi - done < ${models_fp16_config} + done < ${models_onnx_fp16_config} + + while read line; do + fp16_line_info=${line} + if [[ $fp16_line_info == \#* ]]; then + continue + fi + model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms' + cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms + if [ $? = 0 ]; then + converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file} + else + converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1 + fi + done < ${models_caffe_fp16_config} + + while read line; do + fp16_line_info=${line} + if [[ $fp16_line_info == \#* ]]; then + continue + fi + model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms' + cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms + if [ $? = 0 ]; then + converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file} + else + converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1 + fi + done < ${models_tflite_fp16_config} # Convert tflite weightquant models: while read line; do @@ -951,11 +981,45 @@ function Run_arm64() { else run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi + done < ${models_onnx_fp16_config} + + while read line; do + fp16_line_info=${line} + if [[ $fp16_line_info == \#* ]]; then + continue + fi + model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'` + echo "---------------------------------------------------------" >> "${run_arm64_log_file}" + echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_log_file}" - # run benchmark test without clib data echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt - echo './benchmark --modelFile='${model_name}'.fp16.ms --warmUpLoopCount=1 --loopCount=2 --enableFp16=true' >> adb_run_cmd.txt + echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt + + cat adb_run_cmd.txt >> "${run_arm64_log_file}" + adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}" + if [ $? = 0 ]; then + run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file} + else + run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 + fi + done < ${models_caffe_fp16_config} + + while read line; do + fp16_line_info=${line} + if [[ $fp16_line_info == \#* ]]; then + continue + fi + model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'` + accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'` + echo "---------------------------------------------------------" >> "${run_arm64_log_file}" + echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_log_file}" + + echo 'cd /data/local/tmp/benchmark_test' > adb_run_cmd.txt + echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt + echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt + cat adb_run_cmd.txt >> "${run_arm64_log_file}" adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}" if [ $? = 0 ]; then @@ -963,7 +1027,7 @@ function Run_arm64() { else run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1 fi - done < ${models_fp16_config} + done < ${models_tflite_fp16_config} # Run tflite aware training quantization converted models: while read line; do @@ -1302,7 +1366,9 @@ models_tflite_posttraining_config=${basepath}/models_tflite_posttraining.cfg models_caffe_posttraining_config=${basepath}/models_caffe_posttraining.cfg models_tflite_weightquant_config=${basepath}/models_tflite_weightquant.cfg models_onnx_config=${basepath}/models_onnx.cfg -models_fp16_config=${basepath}/models_fp16.cfg +models_onnx_fp16_config=${basepath}/models_onnx_fp16.cfg +models_caffe_fp16_config=${basepath}/models_caffe_fp16.cfg +models_tflite_fp16_config=${basepath}/models_tflite_fp16.cfg models_mindspore_config=${basepath}/models_mindspore.cfg models_mindspore_train_config=${basepath}/models_mindspore_train.cfg models_mindspore_mixbit_config=${basepath}/models_mindspore_mixbit.cfg