!8773 add CI of fp16 ops model

From: @liuwenhao4
Reviewed-by: 
Signed-off-by:

mindspore/lite/test/models_caffe_fp16.cfg

@@ -0,0 +1,68 @@
+age_medium 6
+beard 2
+emotion 60
+gender_res_large_deploy 0.1
+glasses 4
+hat 1
+isface 1
+ml_bank_detect_0312 20
+ml_face_div_parsing 8
+ml_hardware_eyeclose 0.1
+ml_ocr_detect_20200305 10
+Mnet6_0312_extract_pay 15
+pose_3d 90
+RFB-Epoch-170-no-transpose 4
+tracking 4
+mtk_landmark 1
+mtk_pose_tuku 1
+mtk_face_recognition_v1 20
+mtk_2012_ATLANTA_10class_20190614_v41 4
+mtk_detect-deeper-halfdeeper-mbv1-lastearlySSD-shortcut-400-400_nopostprocess_simplified 4
+detect-deeper-halfdeeper-mbv1-shortcut-400-400_nopostprocess_simplified 1
+hiai_face_detect_rfb 4
+hiai_face_isface 0.1
+hiai_face_landmark 0.2
+hiai_face_pose_tuku 1.3
+ml_hand_detection 8
+ml_ocr_cn 6
+ml_ocr_sfz_detect_0325 3
+ml_hardware_liveness 3
+ml_liveness_detect_landmark 1
+ml_face_contour 0.5
+2012_ATLANTA_1class_20190621_v4.x_nomean 1
+ml_ocr_sfz_add_final_0325 0.1
+ml_hardware_pose 2
+ml_bank_recog 0.1
+2012_ATLANTA_10class_20190131_v4.0 12
+mnet 9
+recognition 10
+ml_face_landmark 1
+model_hebing_3branch 40
+hiai_cv_focusShootOCRModel_07 3
+hiai_cv_focusShootOCRModel_03 60
+hiai_cv_focusShootOCRModel_01 14
+hiai_face_hat1 1
+hiai_cv_focusShootOCRModel_04 8
+hiai_cv_focusShootOCRModel_06 13
+hiai_cpu_face_hat 0.3
+hiai_video_seg 1
+hiai_semantic_seg 3
+hiai_human_seg 28
+hiai_face_recognition_1 10
+hiai_cpu_face_detect 4
+hiai_cpu_face_attr 12
+hiai_face_attr1 12
+detect-mbv1-shortcut-400-400_nopostprocess_simplified 8
+detect_mbv1_640_480_nopostprocess_simplified 6
+retinaface 6
+deconv_test_model 20
+deconvs_model 1
+HWSR-s_256_256 10
+age_new 22
+detection_retinaface_fix 13
+landmark 1
+plat_isface 6
+PoseNet_dla_17_x512 5
+ml_location_scene_division 8
+ml_tabel_recog 0.1
+ml_text_division 12

mindspore/lite/test/models_fp16.cfg

@@ -1,34 +0,0 @@
-hiai_cpu_face_detect 4
-#hiai_cpu_face_attr 5000
-hiai_cpu_face_hat 0.3
-hiai_face_detect_rfb 4
-hiai_face_isface 0.1
-hiai_face_landmark 0.2
-hiai_face_pose_tuku 1.3
-ml_face_contour 0.5
-ml_face_landmark 1
-mtk_isface 0.2
-mtk_landmark 0.3
-mtk_pose_tuku 1
-mtk_age_gender.tflite 0.1
-mtk_model_face_dress.tflite 1
-mtk_model_ckpt.tflite 20
-mtk_face_features_v1.tflite 20
-mtk_new_detect.tflite 3
-mtk_pose.tflite 2
-mtk_model_emotions_0727_nosoftmax.tflite 2
-#mtk_model_normalize_object_scene_ps_20200519_f32.tflite 20
-mtk_model_normalize_object_scene_ps_20200826_f32_no_softmax.tflite 22
-mtk_276landmark_0913.tflite 16
-mtk_face_recognition.tflite 8
-mtk_convert_model.tflite 5
-mobilenet_v1_0.5_128.tflite 2.5
-mobilenet_v1_1.0_192.tflite 26
-mobilenet_v2_1.0_224.tflite 3
-retinaface 6
-deconvs_model 1
-efficientnet_lite4_fp32_2.tflite 6
-mnasnet_0.50_224_1_metadata_1.tflite 5
-detect-deeper-halfdeeper-mbv1-shortcut-400-400_nopostprocess_simplified 1
-HWSR-s_256_256 10
-#model_name accuracy_limit

mindspore/lite/test/models_onnx_fp16.cfg

@@ -0,0 +1,34 @@
+mtk_detect-mbv2-shortcut-400-400-simplified.onnx 4
+mtk_emotions-d2012-75.8%.onnx 20
+mtk_face_features_v3.onnx 20
+emotion-ferplus-8.onnx 1
+#rcnn-ilsvrc13-9.onnx 0.1
+efficientnet-lite4-11.onnx 2
+mobilenetv2-7.onnx 8
+shufflenet-v2-10.onnx 5
+squeezenet1.1-7.onnx 1
+densenet-9.onnx 6
+googlenet-9.onnx 3
+inception-v1-9.onnx 3
+inception-v2-9.onnx 4
+shufflenet-9.onnx 1000
+squeezenet1.0-9.onnx 5
+residual_distill_cifar10_bs_1.onnx 1
+residual_distill_cifar10_bs_32.onnx 2
+residual_distill_bs_1.onnx 6
+residual_distill_bs_32.onnx 30
+ml_face_3d.onnx 2
+gts_version-RFB-320_simplified.onnx 3
+mnist-8.onnx 10
+#crnn_lite_lstm_v2.onnx 0.1
+#psenet_lite_mbv2.onnx 0.1
+#super-resolution-10.onnx 1
+#tinyyolov2-8.onnx 0.3
+ml_2012_ocr_cn.onnx 200
+#ml_2012_ocr_cn_noLSTM.onnx 0.1
+candy-9.onnx 5
+mosaic-9.onnx 4
+pointilism-9.onnx 3
+rain-princess-9.onnx 5
+udnie-9.onnx 3
+adversarial_pruning.onnx 3

mindspore/lite/test/models_tflite_fp16.cfg

@@ -0,0 +1,134 @@
+hiai_model_0909_kd_rot_ps_softmax.tflite 10
+hiai_chinese_english_recognize_model_float32.tflite 13
+hiai_bigmodel_ghost_2_1_no_normalized_no_trans_tflite.tflite 10
+hiai_bigmodel_ghost_5_1_no_normalized_no_trans_tflite.tflite 10
+hiai_cn_recognize_modify_padv2.tflite 14
+hiai_model_normalize_object_scene_ps_20200519.tflite 20
+#hiai_detectmodel_06_23_960_480_1180700.tflite 20
+#hiai_detect_curve_model_float32.tflite 20
+#hiai_detectmodel_desnet_256_128_64_32.tflite 20
+mtk_AADB_HADB_MBV2_model_fp32.tflite 5
+mtk_AADB_HADB_MBV3_model_fp32.tflite 6
+mobilenet_v1_0.25_128.tflite 5
+mobilenet_v1_0.25_160.tflite 5
+mobilenet_v1_0.25_192.tflite 3
+mobilenet_v1_0.25_224.tflite 3
+mobilenet_v1_0.5_128.tflite 2.5
+mobilenet_v1_0.5_160.tflite 5
+mobilenet_v1_0.5_192.tflite 6
+mobilenet_v1_0.5_224.tflite 5
+mobilenet_v1_0.75_128.tflite 6
+mobilenet_v1_0.75_160.tflite 6
+mobilenet_v1_0.75_192.tflite 6
+mobilenet_v1_0.75_224.tflite 6
+mobilenet_v1_1.0_128.tflite 10
+mobilenet_v1_1.0_160.tflite 6
+mobilenet_v1_1.0_192.tflite 26
+mobilenet_v1_1.0_224.tflite 6
+mobilenet_v2_1.0_224.tflite 3
+mtk_model_normalize_object_scene_ps_20200519_f32.tflite 20
+mtk_model_ckpt.tflite 20
+mtk_age_gender.tflite 0.1
+mtk_model_face_dress.tflite 2
+mtk_face_features_v1.tflite 20
+densenet.tflite 36
+squeezenet.tflite 6
+resnet_v2_101_299.tflite 3
+mnasnet_1.3_224.tflite 12
+inception_v3.tflite 3
+deeplabv3_257_mv_gpu.tflite 3
+multi_person_mobilenet_v1_075_float.tflite 9
+#hiai_vad.tflite 20
+ide_label_base.tflite 22
+ide_label_retrained.tflite 80
+ml_ei_headpose.tflite 3
+ml_ei_landmark.tflite 3
+mnist.tflite 4
+mobilenet.tflite 0.1
+resnet.tflite 120
+scan_hms_angle1.tflite 4
+scan_hms_detect.tflite 12
+hiai_latin_ocr.tflite 45
+hiai_latin_ocr_1.tflite 13
+ml_ocr_jk.tflite 2
+nasnet_mobile.tflite 3
+nasnet_large.tflite 3
+inception_resnet_v2.tflite 10
+ml_ocr_latin.tflite 15
+hiai_PoseEstimation_Pcm.tflite 15
+hiai_ssd_mobilenetv2_object.tflite 60
+hiai_cv_focusShootOCRModel_02.tflite 13
+hiai_cv_poseEstimation.tflite 190
+inception_v4.tflite 10
+mtk_model_normalize_object_scene_ps_20200519_f16.tflite 10
+mtk_model_face_dress_fp16.tflite 45
+mtk_AADB_HADB_MBV2_model_f16.tflite 5
+mtk_AADB_HADB_MBV3_model_f16.tflite 10
+mtk_model_emotions_0725_fp16.tflite 3
+mtk_face_features_v1_fp16.tflite 20
+siteAI_digcom_AI_ECN.tflite 0.1
+siteAI_digcom_g2v_keras.tflite 5
+siteAI_trans_nonlinear.tflite 0.1
+siteAI_trans_tcpclassify.tflite 5
+siteAI_wireless_depress_w.tflite 8
+siteAI_wireless_restore_w.tflite 0.1
+magenta_arbitrary-image-stylization-v1-256_fp16_prediction_1.tflite 5
+ml_object_detect.tflite 3
+ml_object_detect_1.tflite 5
+hiai_cpu_face_emotion.tflite 6
+hiai_cpu_face_gazing.tflite 5
+hiai_cpu_face_headpose.tflite 100
+hiai_humanDetection.tflite 15
+#hiai_cv_focusShootOCRModel_08.tflite 0.3
+#ml_face_openclose.tflite 40
+hiai_face_model_npu.tflite 5
+hiai_ctpn_feature_map.tflite 10
+hiai_cv_labelDetectorModel_v2.tflite 30
+#hiai_cv_labelDetectorModel_v3.tflite 20
+hiai_cv_labelDetectorModel_v4.tflite 3
+hiai_dress_detect.tflite 3
+hiai_cv_saliencyDetectorModel.tflite 3
+hiai_frozen_inference_graph.tflite 5
+hiai_ghostnet.tflite 3
+#hiai_iMaxDN_RGB.tflite 0.4
+#hiai_iMaxSR_RGB.tflite 1
+hiai_label_and_video.tflite 10
+hiai_lm_inference_graph.tflite 3
+efficientnet_lite0_fp32_2.tflite 6
+efficientnet_lite1_fp32_2.tflite 6
+efficientnet_lite2_fp32_2.tflite 3
+efficientnet_lite3_fp32_2.tflite 6
+efficientnet_lite4_fp32_2.tflite 6
+mnasnet_0.50_224_1_metadata_1.tflite 5
+mnasnet_0.75_224_1_metadata_1.tflite 6
+mnasnet_1.0_128_1_metadata_1.tflite 6
+mnasnet_1.0_160_1_metadata_1.tflite 6
+mnasnet_1.0_192_1_metadata_1.tflite 8
+mnasnet_1.0_224_1_metadata_1.tflite 6
+mnasnet_1.0_96_1_metadata_1.tflite 6
+# ml_vision_guide_detection1.tflite 20
+# ml_vision_guide_detection3.tflite 20
+lite-model_on_device_vision_classifier_popular_us_products_V1_1.tflite 16
+lite-model_on_device_vision_classifier_popular_wine_V1_1.tflite 80
+posenet_mobilenet_float_075_1_default_1.tflite 45
+deeplabv3_1_default_1.tflite 6
+lite-model_deeplabv3-mobilenetv2_dm05-float16_1_default_1.tflite 13
+lite-model_deeplabv3-mobilenetv2-float16_1_default_1.tflite 60
+lite-model_east-text-detector_fp16_1.tflite 60
+lite-model_cartoongan_fp16_1.tflite 3
+lite-model_arbitrary-image-stylization-inceptionv3_fp16_predict_1.tflite 6
+gts_detect_5k_tf115.tflite 6
+mtk_isface.tflite 0.2
+mtk_landmark.tflite 0.1
+mtk_new_detect.tflite 3
+mtk_pose.tflite 2
+mtk_model_emotions_0727_nosoftmax.tflite 2
+mtk_model_normalize_object_scene_ps_20200826_f32_no_softmax.tflite 22
+mtk_276landmark_0913.tflite 16
+mtk_face_recognition.tflite 8
+mtk_convert_model.tflite 5
+smartreply.tflite 0.1
+mindspore_text_classification_tflite.tflite 4
+#ml_location.tflite 0.1
+ml_text_correction.tflite 1
+#ml_pic_shopping.tflite 0.1

mindspore/lite/test/run_benchmark_nets.sh

@@ -164,7 +164,37 @@ function Run_Converter() {
         else
             converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1
         fi
-    done < ${models_fp16_config}
+    done < ${models_onnx_fp16_config}
+
+    while read line; do
+        fp16_line_info=${line}
+        if [[ $fp16_line_info == \#* ]]; then
+          continue
+        fi
+        model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`
+        echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'
+        cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms
+        if [ $? = 0 ]; then
+            converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}
+        else
+            converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1
+        fi
+    done < ${models_caffe_fp16_config}
+
+    while read line; do
+        fp16_line_info=${line}
+        if [[ $fp16_line_info == \#* ]]; then
+          continue
+        fi
+        model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`
+        echo 'cp '${ms_models_path}'/'${model_name}'.ms' ${ms_models_path}'/'${model_name}'.fp16.ms'
+        cp ${ms_models_path}/${model_name}.ms ${ms_models_path}/${model_name}.fp16.ms
+        if [ $? = 0 ]; then
+            converter_result='converter fp16 '${model_name}' pass';echo ${converter_result} >> ${run_converter_result_file}
+        else
+            converter_result='converter fp16 '${model_name}' failed';echo ${converter_result} >> ${run_converter_result_file};return 1
+        fi
+    done < ${models_tflite_fp16_config}
 
     # Convert tflite weightquant models:
     while read line; do
@@ -951,11 +981,45 @@ function Run_arm64() {
         else
             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1
         fi
+    done < ${models_onnx_fp16_config}
+
+    while read line; do
+        fp16_line_info=${line}
+        if [[ $fp16_line_info == \#* ]]; then
+          continue
+        fi
+        model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`
+        accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`
+        echo "---------------------------------------------------------" >> "${run_arm64_log_file}"
+        echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_log_file}"
 
-        # run benchmark test without clib data
         echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt
         echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt
-        echo './benchmark --modelFile='${model_name}'.fp16.ms --warmUpLoopCount=1 --loopCount=2 --enableFp16=true' >> adb_run_cmd.txt
+        echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt
+
+        cat adb_run_cmd.txt >> "${run_arm64_log_file}"
+        adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}"
+        if [ $? = 0 ]; then
+            run_result='arm64_fp16: '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_result_file}
+        else
+            run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1
+        fi
+    done < ${models_caffe_fp16_config}
+
+    while read line; do
+        fp16_line_info=${line}
+        if [[ $fp16_line_info == \#* ]]; then
+          continue
+        fi
+        model_name=`echo ${fp16_line_info}|awk -F ' ' '{print $1}'`
+        accuracy_limit=`echo ${fp16_line_info}|awk -F ' ' '{print $2}'`
+        echo "---------------------------------------------------------" >> "${run_arm64_log_file}"
+        echo "fp16 run: ${model_name}, accuracy limit:${accuracy_limit}" >> "${run_arm64_log_file}"
+
+        echo 'cd  /data/local/tmp/benchmark_test' > adb_run_cmd.txt
+        echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/benchmark_test' >> adb_run_cmd.txt
+        echo './benchmark --modelFile='${model_name}'.fp16.ms --inDataFile=/data/local/tmp/input_output/input/'${model_name}'.ms.bin --benchmarkDataFile=/data/local/tmp/input_output/output/'${model_name}'.ms.out --enableFp16=true --accuracyThreshold='${accuracy_limit} >> adb_run_cmd.txt
+
         cat adb_run_cmd.txt >> "${run_arm64_log_file}"
         adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_arm64_log_file}"
         if [ $? = 0 ]; then
@@ -963,7 +1027,7 @@ function Run_arm64() {
         else
             run_result='arm64_fp16: '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_result_file}; return 1
         fi
-    done < ${models_fp16_config}
+    done < ${models_tflite_fp16_config}
 
     # Run tflite aware training quantization converted models:
     while read line; do
@@ -1302,7 +1366,9 @@ models_tflite_posttraining_config=${basepath}/models_tflite_posttraining.cfg
 models_caffe_posttraining_config=${basepath}/models_caffe_posttraining.cfg
 models_tflite_weightquant_config=${basepath}/models_tflite_weightquant.cfg
 models_onnx_config=${basepath}/models_onnx.cfg
-models_fp16_config=${basepath}/models_fp16.cfg
+models_onnx_fp16_config=${basepath}/models_onnx_fp16.cfg
+models_caffe_fp16_config=${basepath}/models_caffe_fp16.cfg
+models_tflite_fp16_config=${basepath}/models_tflite_fp16.cfg
 models_mindspore_config=${basepath}/models_mindspore.cfg
 models_mindspore_train_config=${basepath}/models_mindspore_train.cfg
 models_mindspore_mixbit_config=${basepath}/models_mindspore_mixbit.cfg