From: @sishuikang Reviewed-by: @zhanghaibo5,@zhang_xue_tong Signed-off-by: @zhang_xue_tongpull/14245/MERGE
| @@ -286,7 +286,7 @@ public class MainActivity extends AppCompatActivity implements OnBackgroundImage | |||||
| progressBar.setVisibility(View.VISIBLE); | progressBar.setVisibility(View.VISIBLE); | ||||
| new Thread(() -> { | new Thread(() -> { | ||||
| isRunningModel = true; | isRunningModel = true; | ||||
| modelTrackingResult = trackingMobile.execut(originBitmap); | |||||
| modelTrackingResult = trackingMobile.execute(originBitmap); | |||||
| if (modelTrackingResult != null) { | if (modelTrackingResult != null) { | ||||
| isRunningModel = false; | isRunningModel = false; | ||||
| lastOriginBitmap = originBitmap; | lastOriginBitmap = originBitmap; | ||||
| @@ -84,19 +84,19 @@ public class TrackingMobile { | |||||
| } | } | ||||
| msConfig.free(); | msConfig.free(); | ||||
| // Complile graph. | |||||
| // Complie graph. | |||||
| if (!session.compileGraph(model)) { | if (!session.compileGraph(model)) { | ||||
| Log.e(TAG, "Compile graph failed"); | Log.e(TAG, "Compile graph failed"); | ||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| return; | return; | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be complie graph again. | |||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| } | } | ||||
| public ModelTrackingResult execut(Bitmap bitmap) { | |||||
| public ModelTrackingResult execute(Bitmap bitmap) { | |||||
| // Set input tensor values. | // Set input tensor values. | ||||
| List<MSTensor> inputs = session.getInputs(); | List<MSTensor> inputs = session.getInputs(); | ||||
| if (inputs.size() != 1) { | if (inputs.size() != 1) { | ||||
| @@ -135,8 +135,8 @@ public class TrackingMobile { | |||||
| int batch = output.getShape()[0]; | int batch = output.getShape()[0]; | ||||
| int channel = output.getShape()[1]; | int channel = output.getShape()[1]; | ||||
| int weight = output.getShape()[2]; | int weight = output.getShape()[2]; | ||||
| int hight = output.getShape()[3]; | |||||
| int plane = weight * hight; | |||||
| int height = output.getShape()[3]; | |||||
| int plane = weight * height; | |||||
| for (int n = 0; n < batch; n++) { | for (int n = 0; n < batch; n++) { | ||||
| for (int c = 0; c < channel; c++) { | for (int c = 0; c < channel; c++) { | ||||
| @@ -187,14 +187,14 @@ The inference code process of bone detection demo is as follows. For details abo | |||||
| - Load the model file and build a computational graph for inference. | - Load the model file and build a computational graph for inference. | ||||
| ```java | ```java | ||||
| // Complile graph. | |||||
| // Compile graph. | |||||
| if (!session.compileGraph(model)) { | if (!session.compileGraph(model)) { | ||||
| Log.e("MS_LITE", "Compile graph failed"); | Log.e("MS_LITE", "Compile graph failed"); | ||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| return false; | return false; | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be compile graph again. | |||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| ``` | ``` | ||||
| @@ -190,14 +190,14 @@ | |||||
| - 加载模型文件并构建用于推理的计算图 | - 加载模型文件并构建用于推理的计算图 | ||||
| ```java | ```java | ||||
| // Complile graph. | |||||
| // Compile graph. | |||||
| if (!session.compileGraph(model)) { | if (!session.compileGraph(model)) { | ||||
| Log.e("MS_LITE", "Compile graph failed"); | Log.e("MS_LITE", "Compile graph failed"); | ||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| return false; | return false; | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be compile graph again. | |||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| ``` | ``` | ||||
| @@ -115,14 +115,14 @@ public class Posenet { | |||||
| } | } | ||||
| msConfig.free(); | msConfig.free(); | ||||
| // Complile graph. | |||||
| // Compile graph. | |||||
| if (!session.compileGraph(model)) { | if (!session.compileGraph(model)) { | ||||
| Log.e("MS_LITE", "Compile graph failed"); | Log.e("MS_LITE", "Compile graph failed"); | ||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| return false; | return false; | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be compile graph again. | |||||
| model.freeBuffer(); | model.freeBuffer(); | ||||
| return true; | return true; | ||||
| @@ -198,60 +198,12 @@ public class Posenet { | |||||
| String.format("Interpreter took %.2f ms", 1.0f * lastInferenceTimeNanos / 1_000_000) | String.format("Interpreter took %.2f ms", 1.0f * lastInferenceTimeNanos / 1_000_000) | ||||
| ); | ); | ||||
| // Get output tensor values. | |||||
| List<MSTensor> heatmaps_list = session.getOutputsByNodeName("Conv2D-27"); | |||||
| if (heatmaps_list == null) { | |||||
| return null; | |||||
| } | |||||
| MSTensor heatmaps_tensors = heatmaps_list.get(0); | |||||
| float[][][][] heatmaps = runConv2Dfor27(); | |||||
| float[][][][] offsets = runConv2Dfor28(); | |||||
| float[] heatmaps_results = heatmaps_tensors.getFloatData(); | |||||
| int[] heatmapsShape = heatmaps_tensors.getShape(); //1, 9, 9 ,17 | |||||
| float[][][][] heatmaps = new float[heatmapsShape[0]][][][]; | |||||
| for (int x = 0; x < heatmapsShape[0]; x++) { // heatmapsShape[0] =1 | |||||
| float[][][] arrayThree = new float[heatmapsShape[1]][][]; | |||||
| for (int y = 0; y < heatmapsShape[1]; y++) { // heatmapsShape[1] = 9 | |||||
| float[][] arrayTwo = new float[heatmapsShape[2]][]; | |||||
| for (int z = 0; z < heatmapsShape[2]; z++) { //heatmapsShape[2] = 9 | |||||
| float[] arrayOne = new float[heatmapsShape[3]]; //heatmapsShape[3] = 17 | |||||
| for (int i = 0; i < heatmapsShape[3]; i++) { | |||||
| int n = i + z * heatmapsShape[3] + y * heatmapsShape[2] * heatmapsShape[3] + x * heatmapsShape[1] * heatmapsShape[2] * heatmapsShape[3]; | |||||
| arrayOne[i] = heatmaps_results[n]; //1*9*9*17 ?? | |||||
| } | |||||
| arrayTwo[z] = arrayOne; | |||||
| } | |||||
| arrayThree[y] = arrayTwo; | |||||
| } | |||||
| heatmaps[x] = arrayThree; | |||||
| } | |||||
| List<MSTensor> offsets_list = session.getOutputsByNodeName("Conv2D-28"); | |||||
| if (offsets_list == null) { | |||||
| if (heatmaps == null || offsets ==null){ | |||||
| return null; | return null; | ||||
| } | } | ||||
| MSTensor offsets_tensors = offsets_list.get(0); | |||||
| float[] offsets_results = offsets_tensors.getFloatData(); | |||||
| int[] offsetsShapes = offsets_tensors.getShape(); | |||||
| float[][][][] offsets = new float[offsetsShapes[0]][][][]; | |||||
| for (int x = 0; x < offsetsShapes[0]; x++) { | |||||
| float[][][] offsets_arrayThree = new float[offsetsShapes[1]][][]; | |||||
| for (int y = 0; y < offsetsShapes[1]; y++) { | |||||
| float[][] offsets_arrayTwo = new float[offsetsShapes[2]][]; | |||||
| for (int z = 0; z < offsetsShapes[2]; z++) { | |||||
| float[] offsets_arrayOne = new float[offsetsShapes[3]]; | |||||
| for (int i = 0; i < offsetsShapes[3]; i++) { | |||||
| int n = i + z * offsetsShapes[3] + y * offsetsShapes[2] * offsetsShapes[3] + x * offsetsShapes[1] * offsetsShapes[2] * offsetsShapes[3]; | |||||
| offsets_arrayOne[i] = offsets_results[n]; | |||||
| } | |||||
| offsets_arrayTwo[z] = offsets_arrayOne; | |||||
| } | |||||
| offsets_arrayThree[y] = offsets_arrayTwo; | |||||
| } | |||||
| offsets[x] = offsets_arrayThree; | |||||
| } | |||||
| int height = ((Object[]) heatmaps[0]).length; //9 | int height = ((Object[]) heatmaps[0]).length; //9 | ||||
| int width = ((Object[]) heatmaps[0][0]).length; //9 | int width = ((Object[]) heatmaps[0][0]).length; //9 | ||||
| @@ -288,8 +240,8 @@ public class Posenet { | |||||
| int positionY = (int) position.first; | int positionY = (int) position.first; | ||||
| int positionX = (int) position.second; | int positionX = (int) position.second; | ||||
| yCoords[i] = ((float) positionY / (float) (height - 1) * bitmap.getHeight() + offsets[0][positionY][positionX][i]); | |||||
| xCoords[i] = ((float) positionX / (float) (width - 1) * bitmap.getWidth() + offsets[0][positionY][positionX][i + numKeypoints]); | |||||
| yCoords[i] = ((float) positionY / (float) (height - 1) * bitmap.getHeight() + offsets[0][positionY][positionX][i]); | |||||
| xCoords[i] = ((float) positionX / (float) (width - 1) * bitmap.getWidth() + offsets[0][positionY][positionX][i + numKeypoints]); | |||||
| confidenceScores[i] = sigmoid(heatmaps[0][positionY][positionX][i]); | confidenceScores[i] = sigmoid(heatmaps[0][positionY][positionX][i]); | ||||
| } | } | ||||
| @@ -311,4 +263,72 @@ public class Posenet { | |||||
| return person; | return person; | ||||
| } | } | ||||
| private float[][][][] runConv2Dfor27() { | |||||
| // Get output tensor values. | |||||
| List<MSTensor> heatmaps_list = session.getOutputsByNodeName("Conv2D-27"); | |||||
| if (heatmaps_list == null) { | |||||
| return null; | |||||
| } | |||||
| MSTensor heatmaps_tensors = heatmaps_list.get(0); | |||||
| float[] heatmaps_results = heatmaps_tensors.getFloatData(); | |||||
| int[] heatmapsShape = heatmaps_tensors.getShape(); //1, 9, 9 ,17 | |||||
| if (heatmapsShape[0] < 0 || heatmapsShape[1] < 0 || heatmapsShape[2] < 0 || heatmapsShape[3] < 0) { | |||||
| return null; | |||||
| } | |||||
| float[][][][] heatmaps = new float[heatmapsShape[0]][][][]; | |||||
| for (int x = 0; x < heatmapsShape[0]; x++) { // heatmapsShape[0] =1 | |||||
| float[][][] arrayThree = new float[heatmapsShape[1]][][]; | |||||
| for (int y = 0; y < heatmapsShape[1]; y++) { // heatmapsShape[1] = 9 | |||||
| float[][] arrayTwo = new float[heatmapsShape[2]][]; | |||||
| for (int z = 0; z < heatmapsShape[2]; z++) { //heatmapsShape[2] = 9 | |||||
| float[] arrayOne = new float[heatmapsShape[3]]; //heatmapsShape[3] = 17 | |||||
| for (int i = 0; i < heatmapsShape[3]; i++) { | |||||
| int n = i + z * heatmapsShape[3] + y * heatmapsShape[2] * heatmapsShape[3] + x * heatmapsShape[1] * heatmapsShape[2] * heatmapsShape[3]; | |||||
| arrayOne[i] = heatmaps_results[n]; //1*9*9*17 ?? | |||||
| } | |||||
| arrayTwo[z] = arrayOne; | |||||
| } | |||||
| arrayThree[y] = arrayTwo; | |||||
| } | |||||
| heatmaps[x] = arrayThree; | |||||
| } | |||||
| return heatmaps; | |||||
| } | |||||
| private float[][][][] runConv2Dfor28() { | |||||
| List<MSTensor> offsets_list = session.getOutputsByNodeName("Conv2D-28"); | |||||
| if (offsets_list == null) { | |||||
| return null; | |||||
| } | |||||
| MSTensor offsets_tensors = offsets_list.get(0); | |||||
| float[] offsets_results = offsets_tensors.getFloatData(); | |||||
| int[] offsetsShapes = offsets_tensors.getShape(); | |||||
| if (offsetsShapes[0] < 0 || offsetsShapes[1] < 0 || offsetsShapes[2] < 0 || offsetsShapes[3] < 0) { | |||||
| return null; | |||||
| } | |||||
| float[][][][] offsets = new float[offsetsShapes[0]][][][]; | |||||
| for (int x = 0; x < offsetsShapes[0]; x++) { | |||||
| float[][][] offsets_arrayThree = new float[offsetsShapes[1]][][]; | |||||
| for (int y = 0; y < offsetsShapes[1]; y++) { | |||||
| float[][] offsets_arrayTwo = new float[offsetsShapes[2]][]; | |||||
| for (int z = 0; z < offsetsShapes[2]; z++) { | |||||
| float[] offsets_arrayOne = new float[offsetsShapes[3]]; | |||||
| for (int i = 0; i < offsetsShapes[3]; i++) { | |||||
| int n = i + z * offsetsShapes[3] + y * offsetsShapes[2] * offsetsShapes[3] + x * offsetsShapes[1] * offsetsShapes[2] * offsetsShapes[3]; | |||||
| offsets_arrayOne[i] = offsets_results[n]; | |||||
| } | |||||
| offsets_arrayTwo[z] = offsets_arrayOne; | |||||
| } | |||||
| offsets_arrayThree[y] = offsets_arrayTwo; | |||||
| } | |||||
| offsets[x] = offsets_arrayThree; | |||||
| } | |||||
| return offsets; | |||||
| } | |||||
| } | } | ||||
| @@ -154,7 +154,7 @@ The inference code process of style transfer demo is as follows. For details abo | |||||
| - Load the model file and build a computational graph for inference. | - Load the model file and build a computational graph for inference. | ||||
| ```java | ```java | ||||
| // Complile graph. | |||||
| // Compile graph. | |||||
| if (!Predict_session.compileGraph(style_predict_model)) { | if (!Predict_session.compileGraph(style_predict_model)) { | ||||
| Log.e("MS_LITE", "Compile style_predict graph failed"); | Log.e("MS_LITE", "Compile style_predict graph failed"); | ||||
| style_predict_model.freeBuffer(); | style_predict_model.freeBuffer(); | ||||
| @@ -164,7 +164,7 @@ The inference code process of style transfer demo is as follows. For details abo | |||||
| style_transform_model.freeBuffer(); | style_transform_model.freeBuffer(); | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be compile graph again. | |||||
| style_predict_model.freeBuffer(); | style_predict_model.freeBuffer(); | ||||
| style_transform_model.freeBuffer(); | style_transform_model.freeBuffer(); | ||||
| ``` | ``` | ||||
| @@ -223,7 +223,7 @@ The inference code process of style transfer demo is as follows. For details abo | |||||
| // Get output tensor values. | // Get output tensor values. | ||||
| List<String> tensorNames = Predict_session.getOutputTensorNames(); | List<String> tensorNames = Predict_session.getOutputTensorNames(); | ||||
| Map<String, MSTensor> outputs = Predict_session.getOutputMapByTensor(); | Map<String, MSTensor> outputs = Predict_session.getOutputMapByTensor(); | ||||
| Set<Map.Entry<String, MSTensor>> entrys = outputs.entrySet(); | |||||
| Set<Map.Entry<String, MSTensor>> entry = outputs.entrySet(); | |||||
| float[] Predict_results = null; | float[] Predict_results = null; | ||||
| for (String tensorName : tensorNames) { | for (String tensorName : tensorNames) { | ||||
| @@ -157,7 +157,7 @@ | |||||
| - 加载模型文件并构建用于推理的计算图 | - 加载模型文件并构建用于推理的计算图 | ||||
| ```java | ```java | ||||
| // Complile graph. | |||||
| // Compile graph. | |||||
| if (!Predict_session.compileGraph(style_predict_model)) { | if (!Predict_session.compileGraph(style_predict_model)) { | ||||
| Log.e("MS_LITE", "Compile style_predict graph failed"); | Log.e("MS_LITE", "Compile style_predict graph failed"); | ||||
| style_predict_model.freeBuffer(); | style_predict_model.freeBuffer(); | ||||
| @@ -167,7 +167,7 @@ | |||||
| style_transform_model.freeBuffer(); | style_transform_model.freeBuffer(); | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be compile graph again. | |||||
| style_predict_model.freeBuffer(); | style_predict_model.freeBuffer(); | ||||
| style_transform_model.freeBuffer(); | style_transform_model.freeBuffer(); | ||||
| ``` | ``` | ||||
| @@ -225,7 +225,7 @@ | |||||
| // Get output tensor values. | // Get output tensor values. | ||||
| List<String> tensorNames = Predict_session.getOutputTensorNames(); | List<String> tensorNames = Predict_session.getOutputTensorNames(); | ||||
| Map<String, MSTensor> outputs = Predict_session.getOutputMapByTensor(); | Map<String, MSTensor> outputs = Predict_session.getOutputMapByTensor(); | ||||
| Set<Map.Entry<String, MSTensor>> entrys = outputs.entrySet(); | |||||
| Set<Map.Entry<String, MSTensor>> entry = outputs.entrySet(); | |||||
| float[] Predict_results = null; | float[] Predict_results = null; | ||||
| for (String tensorName : tensorNames) { | for (String tensorName : tensorNames) { | ||||
| @@ -96,7 +96,7 @@ public class StyleTransferModelExecutor { | |||||
| msConfig.free(); | msConfig.free(); | ||||
| // Complile graph. | |||||
| // Compile graph. | |||||
| if (!Predict_session.compileGraph(style_predict_model)) { | if (!Predict_session.compileGraph(style_predict_model)) { | ||||
| Log.e("MS_LITE", "Compile style_predict graph failed"); | Log.e("MS_LITE", "Compile style_predict graph failed"); | ||||
| style_predict_model.freeBuffer(); | style_predict_model.freeBuffer(); | ||||
| @@ -106,7 +106,7 @@ public class StyleTransferModelExecutor { | |||||
| style_transform_model.freeBuffer(); | style_transform_model.freeBuffer(); | ||||
| } | } | ||||
| // Note: when use model.freeBuffer(), the model can not be complile graph again. | |||||
| // Note: when use model.freeBuffer(), the model can not be compile graph again. | |||||
| style_predict_model.freeBuffer(); | style_predict_model.freeBuffer(); | ||||
| style_transform_model.freeBuffer(); | style_transform_model.freeBuffer(); | ||||
| } | } | ||||
| @@ -165,7 +165,7 @@ public class StyleTransferModelExecutor { | |||||
| // Get output tensor values. | // Get output tensor values. | ||||
| List<String> tensorNames = Predict_session.getOutputTensorNames(); | List<String> tensorNames = Predict_session.getOutputTensorNames(); | ||||
| Map<String, MSTensor> outputs = Predict_session.getOutputMapByTensor(); | Map<String, MSTensor> outputs = Predict_session.getOutputMapByTensor(); | ||||
| Set<Map.Entry<String, MSTensor>> entrys = outputs.entrySet(); | |||||
| Set<Map.Entry<String, MSTensor>> entry = outputs.entrySet(); | |||||
| float[] Predict_results = null; | float[] Predict_results = null; | ||||
| for (String tensorName : tensorNames) { | for (String tensorName : tensorNames) { | ||||