!9563 tod examples

From: @yonibaehr_admin
Reviewed-by: 
Signed-off-by:

mindspore/lite/example/train_lenet/model/prepare_model.sh

@@ -1,24 +0,0 @@
-CONVERTER="../../../../../mindspore/lite/build/tools/converter/converter_lite"
-if [ ! -f "$CONVERTER" ]; then
-  if ! command -v converter_lite &> /dev/null
-  then
-    echo "converter_lite could not be found in MindSpore build directory nor in system path"
-    exit
-  else
-    CONVERTER=converter_lite
-  fi
-fi
-
-echo "============Exporting=========="
-if [ -n "$1" ]; then
-  DOCKER_IMG=$1
-  docker run -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER --privileged=true ${DOCKER_IMG} /bin/bash -c "python lenet_export.py; chmod 444 lenet_tod.mindir; rm -rf __pycache__"
-else
-  echo "MindSpore docker was not provided, attempting to run locally"
-  python lenet_export.py
-fi
-
-
-echo "============Converting========="
-$CONVERTER --fmk=MINDIR --trainModel=true --modelFile=lenet_tod.mindir --outputFile=lenet_tod
-

mindspore/lite/example/train_lenet/prepare_and_run.sh

@@ -1,82 +0,0 @@
-#!/bin/bash
-
-display_usage() {
-	echo -e "\nUsage: prepare_and_run.sh dataset_path [mindspore_docker] [release.tar.gz]\n"
-	}
-
-if [ -n "$1" ]; then
-  MNIST_DATA_PATH=$1
-else
-  echo "MNIST Dataset directory path was not provided"
-  display_usage
-  exit 0
-fi
-
-if [ -n "$2" ]; then
-  DOCKER=$2
-else
-  DOCKER=""
-  #echo "MindSpore docker was not provided"
-  #display_usage
-  #exit 0
-fi
-
-if [ -n "$3" ]; then
-  TARBALL=$3
-else 
-  if [ -f ../../../../output/mindspore-lite-*-runtime-arm64-cpu-train.tar.gz ]; then
-    TARBALL="../../../../output/mindspore-lite-*-runtime-arm64-cpu-train.tar.gz"
-  else
-    echo "release.tar.gz was not found"
-    display_usage
-    exit 0
-  fi
-fi
-
-
-# Prepare the model
-cd model/
-rm -f *.ms
-./prepare_model.sh $DOCKER 
-cd -
-
-# Copy the .ms model to the package folder
-rm -rf package
-mkdir -p package/model
-cp model/*.ms package/model
-
-# Copy the running script to the package
-cp scripts/train.sh package/
-cp scripts/eval.sh package/
-
-# Copy the shared MindSpore ToD library
-tar -xzvf ${TARBALL} --wildcards --no-anchored libmindspore-lite.so
-tar -xzvf ${TARBALL} --wildcards --no-anchored include
-mv mindspore-*/lib package/
-mkdir msl
-mv mindspore-*/* msl/
-rm -rf mindspore-*
-
-# Copy the dataset to the package
-cp -r ${MNIST_DATA_PATH} package/dataset
-
-# Compile program
-make TARGET=arm64
- 
-# Copy the executable to the package
-mv bin package/
-
-# Push the folder to the device
-adb push package /data/local/tmp/
-
-echo "Training on Device"
-adb shell < scripts/run_train.sh
-
-echo
-echo "Load trained model and evaluate accuracy"
-adb shell < scripts/run_eval.sh
-echo
-
-#rm -rf src/*.o package model/__pycache__ model/*.ms
-
-#./prepare_and_run.sh /opt/share/dataset/mnist mindspore_dev:5

mindspore/lite/example/train_lenet/scripts/run_eval.sh

@@ -1,2 +0,0 @@
-cd /data/local/tmp/package
-/system/bin/sh eval.sh

mindspore/lite/example/train_lenet/scripts/run_train.sh

@@ -1,2 +0,0 @@
-cd /data/local/tmp/package
-/system/bin/sh  train.sh

mindspore/lite/examples/train_lenet/Makefile

@@ -0,0 +1,41 @@
+BASE_DIR=$(realpath ../../../../)
+APP:=bin/net_runner
+MSLIB:=mindspore-lite
+MSDIR:=$(realpath package-$(TARGET)/lib)
+
+SRC:=src/net_runner.cc  src/dataset.cc
+OBJ:=$(SRC:.cc=.o)
+
+CFLAGS := -Ofast -std=c++17  \
+	-I . \
+        -I ./msl \
+        -I ./msl/third_party/flatbuffers/include
+
+
+ifeq ($(TARGET),arm64)
+CXX :=  ${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++
+CFLAGS  += --target=aarch64-none-linux-android21 --gcc-toolchain=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64 --sysroot=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot -fdata-sections -ffunction-sections 
+LDFLAGS := --target=aarch64-none-linux-android21 --gcc-toolchain=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64 --sysroot=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot -Wl,--gc-sections 
+LDFLAGS += -L$(MSDIR) -l$(MSLIB) -pthread -llog -latomic -lm  
+else
+CFLAGS += -g
+LDFLAGS := -L$(MSDIR) -l$(MSLIB) -lpthread -Wl,-rpath,$(MSDIR)
+endif  
+LD := ${CXX}
+
+
+all:$(APP)
+
+$(APP): $(OBJ) $(MSDIR)/lib$(MSLIB).so
+	@mkdir -p bin
+	$(LD) $(OBJ) $(LDFLAGS) -o $@
+
+clean:
+	rm -rf src/*.o bin/ 
+
+
+mrproper: 
+	rm -rf package* msl src/*.o bin/ model/*.mindir model/*.ms model/*.so model/converter_lite
+
+%.o:%.cc
+	$(CXX) $(CFLAGS) -c $< -o $@

mindspore/lite/example/train_lenet/README.md → mindspore/lite/examples/train_lenet/README.md

@@ -21,7 +21,7 @@ LeNet is a very simple network which is composed of only 5 layers, 2 of which ar
 
 # Dataset
 
-In this example we use the MNIST dataset of handwritten digits as published in [THE MNIST DATABASE](<http://yann.lecun.com/exdb/mnist/>)
+In this example we use the MNIST dataset of handwritten digits as published in [THE MNIST DATABASE](http://yann.lecun.com/exdb/mnist/)
 
 - Dataset size：52.4M，60,000 28*28 in 10 classes
     - Test：10,000 images
@@ -31,7 +31,7 @@ In this example we use the MNIST dataset of handwritten digits as published in [
 
 - The dataset directory structure is as follows:
 
-```python
+```text
 mnist/
 ├── test
 │   ├── t10k-images-idx3-ubyte
@@ -54,15 +54,16 @@ mnist/
 
 After installing all the above mentioned, the script in the home directory could be run with the following arguments:
 
-```python
-sh ./prepare_and_run.sh DATASET_PATH [MINDSPORE_DOCKER] [RELEASE.tar.gz]
+```bash
+sh ./prepare_and_run.sh -D DATASET_PATH [-d MINDSPORE_DOCKER] [-r RELEASE.tar.gz] [-t arm64|x86]
 ```
 
 where:
 
 - DATASET_PATH is the path to the [dataset](#dataset),
 - MINDSPORE_DOCKER is the image name of the docker that runs [MindSpore](#environment-requirements). If not provided MindSpore will be run locally
-- and REALEASE.tar.gz is a pointer to the MindSpore ToD release tar ball. If not provided, the script will attempt to find MindSpore ToD compilation output.
+- REALEASE.tar.gz is a pointer to the MindSpore ToD release tar ball. If not provided, the script will attempt to find MindSpore ToD compilation output
+- target is defaulted to arm64, i.e., on-device. If x86 is provided, the demo will be run locally. Note that infrastructure is not optimized for device
 
 # Script Detailed Description
 

mindspore/lite/example/train_lenet/model/lenet_export.py → mindspore/lite/examples/train_lenet/model/lenet_export.py

@@ -14,7 +14,6 @@
 # ============================================================================
 """lenet_export."""
 
-import sys
 from mindspore import context, Tensor
 import mindspore.common.dtype as mstype
 from mindspore.train.serialization import export
@@ -22,15 +21,14 @@ from lenet import LeNet5
 import numpy as np
 from train_utils import TrainWrap
 
-sys.path.append('../../../cv/lenet/src/')
 
 n = LeNet5()
 n.set_train()
 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False)
 
-batch_size = 32
-x = Tensor(np.ones((batch_size, 1, 32, 32)), mstype.float32)
-label = Tensor(np.zeros([batch_size, 10]).astype(np.float32))
+BATCH_SIZE = 32
+x = Tensor(np.ones((BATCH_SIZE, 1, 32, 32)), mstype.float32)
+label = Tensor(np.zeros([BATCH_SIZE, 10]).astype(np.float32))
 net = TrainWrap(n)
 export(net, x, label, file_name="lenet_tod.mindir", file_format='MINDIR')
 

mindspore/lite/examples/train_lenet/model/prepare_model.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+echo "============Exporting=========="
+if [ -n "$1" ]; then
+  DOCKER_IMG=$1
+  docker run -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER --privileged=true ${DOCKER_IMG} /bin/bash -c "PYTHONPATH=../../../../../model_zoo/official/cv/lenet/src python lenet_export.py; chmod 444 lenet_tod.mindir; rm -rf __pycache__"
+else
+  echo "MindSpore docker was not provided, attempting to run locally"
+  PYTHONPATH=../../../../../model_zoo/official/cv/lenet/src python lenet_export.py
+fi
+
+
+CONVERTER="../../../build/tools/converter/converter_lite"
+if [ ! -f "$CONVERTER" ]; then
+  if ! command -v converter_lite &> /dev/null
+  then
+    tar -xzf ../../../../../output/mindspore-lite-*-converter-ubuntu-train.tar.gz --strip-components 2 --wildcards --no-anchored converter_lite libmindspore_gvar.so
+    if [ -f ./converter_lite ]; then
+      CONVERTER=./converter_lite
+    else
+      echo "converter_lite could not be found in MindSpore build directory nor in system path"
+      exit 1
+    fi
+  else
+    CONVERTER=converter_lite
+  fi
+fi
+
+
+echo "============Converting========="
+LD_LIBRARY_PATH=./ $CONVERTER --fmk=MINDIR --trainModel=true --modelFile=lenet_tod.mindir --outputFile=lenet_tod
+

mindspore/lite/example/train_lenet/model/train_utils.py → mindspore/lite/examples/train_lenet/model/train_utils.py

@@ -22,13 +22,13 @@ def TrainWrap(net, loss_fn=None, optimizer=None, weights=None):
     TrainWrap
     """
     if loss_fn is None:
-        loss_fn = nn.SoftmaxCrossEntropyWithLogits()
+        loss_fn = nn.SoftmaxCrossEntropyWithLogits(reduction='mean')
     loss_net = nn.WithLossCell(net, loss_fn)
     loss_net.set_train()
     if weights is None:
         weights = ParameterTuple(net.trainable_params())
     if optimizer is None:
-        optimizer = nn.Adam(weights, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False,
+        optimizer = nn.Adam(weights, learning_rate=1e-2, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False,
                             use_nesterov=False, weight_decay=0.0, loss_scale=1.0)
     train_net = nn.TrainOneStepCell(loss_net, optimizer)
     return train_net

mindspore/lite/examples/train_lenet/prepare_and_run.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+display_usage()
+{
+  echo -e "\nUsage: prepare_and_run.sh -D dataset_path [-d mindspore_docker] [-r release.tar.gz] [-t arm64|x86]\n"
+}
+
+checkopts()
+{
+  TARGET="arm64"
+  DOCKER=""
+  MNIST_DATA_PATH=""
+  while getopts 'D:d:r:t:' opt
+  do
+    OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
+    case "${opt}" in
+      D)
+        MNIST_DATA_PATH=$OPTARG
+        ;;
+      d)
+        DOCKER=$OPTARG
+        ;;
+      t)
+        if [ "$OPTARG" == "arm64" ] || [ "$OPTARG" == "x86" ]; then
+          TARGET=$OPTARG
+        else
+          echo "No such target " $OPTARG
+          display_usage
+          exit 1
+        fi
+        ;;
+      r)
+        TARBALL=$OPTARG
+        ;;
+      *)
+        echo "Unknown option ${opt}!"
+        display_usage
+        exit 1
+    esac
+  done
+}
+
+checkopts "$@"
+if [ "$MNIST_DATA_PATH" == "" ]; then
+  echo "MNIST Dataset directory path was not provided"
+  display_usage
+  exit 1
+fi
+
+if [ "$TARBALL" == "" ]; then
+  file=$(ls ../../../../output/mindspore-lite-*-runtime-${TARGET}-cpu-train.tar.gz)
+  if [ -f ${file} ]; then
+    TARBALL=${file}
+  else
+    echo "release.tar.gz was not found"
+    display_usage
+    exit 1
+  fi
+fi
+
+# Prepare the model
+cd model/ || exit 1
+rm -f *.ms
+./prepare_model.sh $DOCKER || exit 1
+cd ../
+
+# Copy the .ms model to the package folder
+
+PACKAGE=package-${TARGET}
+
+rm -rf ${PACKAGE}
+mkdir -p ${PACKAGE}/model
+cp model/*.ms ${PACKAGE}/model
+
+# Copy the running script to the package
+cp scripts/*.sh ${PACKAGE}/
+
+# Copy the shared MindSpore ToD library
+tar -xzf ${TARBALL} --wildcards --no-anchored libmindspore-lite.so
+tar -xzf ${TARBALL} --wildcards --no-anchored include
+mv mindspore-*/lib ${PACKAGE}/
+rm -rf msl
+mkdir msl
+mv mindspore-*/* msl/
+rm -rf mindspore-*
+
+# Copy the dataset to the package
+cp -r ${MNIST_DATA_PATH} ${PACKAGE}/dataset
+
+echo "==========Compiling============"
+make TARGET=${TARGET}
+ 
+# Copy the executable to the package
+mv bin ${PACKAGE}/ || exit 1
+
+if [ "${TARGET}" == "arm64" ]; then
+  echo "=======Pushing to device======="
+  adb push ${PACKAGE} /data/local/tmp/
+
+  echo "========Training on Device====="
+  adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh train.sh"
+
+  echo
+  echo "===Evaluating trained Model====="
+  adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh eval.sh"
+  echo
+else
+  cd ${PACKAGE} || exit 1
+  echo "======Training Locally========="
+  ./train.sh
+
+  echo "===Evaluating trained Model====="
+  ./eval.sh
+  cd ..
+fi
+

mindspore/lite/example/train_lenet/scripts/eval.sh → mindspore/lite/examples/train_lenet/scripts/eval.sh

@@ -15,5 +15,4 @@
 # ============================================================================
 
 # an simple tutorial as follows, more parameters can be setting
-DATA_PATH=$1
 LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/lenet_tod_trained_3000.ms -e 0 -d dataset 

mindspore/lite/example/train_lenet/scripts/train.sh → mindspore/lite/examples/train_lenet/scripts/train.sh

@@ -15,7 +15,4 @@
 # ============================================================================
 
 # an simple tutorial as follows, more parameters can be setting
-script_self=$(readlink -f "$0")
-self_path=$(dirname "${script_self}")
-DATA_PATH=$1
 LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/lenet_tod.ms -e 3000 -d dataset 

mindspore/lite/example/train_lenet/src/dataset.cc → mindspore/lite/examples/train_lenet/src/dataset.cc

@@ -72,11 +72,8 @@ int DataSet::Init(const std::string &data_base_directory, database_type type) {
 }
 
 void DataSet::InitializeMNISTDatabase(std::string dpath) {
-  //  int total_data = 0;
   num_of_classes_ = 10;
-  //  total_data +=
   ReadMNISTFile(dpath + "/train/train-images-idx3-ubyte", dpath + "/train/train-labels-idx1-ubyte", &train_data_);
-  //  total_data +=
   ReadMNISTFile(dpath + "/test/t10k-images-idx3-ubyte", dpath + "/test/t10k-labels-idx1-ubyte", &test_data_);
 }
 
@@ -152,49 +149,3 @@ int DataSet::ReadMNISTFile(const std::string &ifile_name, const std::string &lfi
   }
   return number_of_labels;
 }
-
-std::vector<FileTuple> DataSet::ReadFileList(std::string dpath) {
-  std::vector<FileTuple> vec;
-  std::ifstream ifs(dpath + "/file_list.txt");
-  std::string file_name;
-  if (ifs.is_open()) {
-    int label;
-    while (!ifs.eof()) {
-      ifs >> label >> file_name;
-      vec.push_back(make_tuple(label, file_name));
-    }
-  }
-  return vec;
-}
-
-std::vector<FileTuple> DataSet::ReadDir(const std::string dpath) {
-  std::filesystem::directory_iterator dir(dpath);
-  std::vector<FileTuple> vec;
-  LabelId label_id;
-  int class_id = 0;
-  int class_label;
-  for (const auto p : dir) {
-    if (p.is_directory()) {
-      std::string path = p.path().stem().string();
-      auto label = label_id.find(path);
-      if (label == label_id.end()) {
-        label_id[path] = class_id;
-        class_label = class_id;
-        class_id++;
-        num_of_classes_ = class_id;
-      } else {
-        class_label = label->second;
-      }
-      std::filesystem::directory_iterator ndir(dpath + "/" + path);
-      for (const auto np : ndir) {
-        if (np.path().extension().string() == ".bin") {
-          std::string entry =
-            dpath + "/" + np.path().parent_path().stem().string() + "/" + np.path().filename().string();
-          FileTuple ft = make_tuple(class_label, entry);
-          vec.push_back(ft);
-        }
-      }
-    }
-  }
-  return vec;
-}

mindspore/lite/example/train_lenet/src/dataset.h → mindspore/lite/examples/train_lenet/src/dataset.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MODEL_ZOO_OFFICIAL_TOD_TRAIN_LENET_SRC_DATASET_H_
-#define MODEL_ZOO_OFFICIAL_TOD_TRAIN_LENET_SRC_DATASET_H_
+#ifndef MINDSPORE_LITE_EXAMPLES_TRAIN_LENET_SRC_DATASET_H_
+#define MINDSPORE_LITE_EXAMPLES_TRAIN_LENET_SRC_DATASET_H_
 
 #include <tuple>
 #include <string>
@@ -42,8 +42,6 @@ class DataSet {
   unsigned int expected_data_size() { return expected_data_size_; }
 
  private:
-  std::vector<FileTuple> ReadFileList(std::string dpath);
-  std::vector<FileTuple> ReadDir(const std::string dpath);
   int ReadMNISTFile(const std::string &ifile, const std::string &lfile, std::vector<DataLabelTuple> *dataset);
   void InitializeMNISTDatabase(std::string dpath);
 
@@ -53,4 +51,4 @@ class DataSet {
   unsigned int expected_data_size_ = 0;
 };
 
-#endif  // MODEL_ZOO_OFFICIAL_TOD_TRAIN_LENET_SRC_DATASET_H_
+#endif  // MINDSPORE_LITE_EXAMPLES_TRAIN_LENET_SRC_DATASET_H_

mindspore/lite/example/train_lenet/src/net_runner.h → mindspore/lite/examples/train_lenet/src/net_runner.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MODEL_ZOO_OFFICIAL_TOD_TRAIN_LENET_SRC_NET_RUNNER_H_
-#define MODEL_ZOO_OFFICIAL_TOD_TRAIN_LENET_SRC_NET_RUNNER_H_
+#ifndef MINDSPORE_LITE_EXAMPLES_TRAIN_LENET_SRC_NET_RUNNER_H_
+#define MINDSPORE_LITE_EXAMPLES_TRAIN_LENET_SRC_NET_RUNNER_H_
 
 #include <tuple>
 #include <filesystem>
@@ -58,4 +58,4 @@ class NetRunner {
   static unsigned int seed_;
 };
 
-#endif  // MODEL_ZOO_OFFICIAL_TOD_TRAIN_LENET_SRC_NET_RUNNER_H_
+#endif  // MINDSPORE_LITE_EXAMPLES_TRAIN_LENET_SRC_NET_RUNNER_H_

mindspore/lite/examples/transfer_learning/Makefile

@@ -0,0 +1,41 @@
+BASE_DIR=$(realpath ../../../../)
+APP:=bin/net_runner
+MSLIB:=mindspore-lite
+MSDIR:=$(realpath package-$(TARGET)/lib)
+
+SRC:=src/net_runner.cc  src/dataset.cc
+OBJ:=$(SRC:.cc=.o)
+
+CFLAGS := -Ofast -std=c++17  \
+	-I . \
+        -I ./msl \
+        -I ./msl/third_party/flatbuffers/include
+
+
+ifeq ($(TARGET),arm64)
+CXX :=  ${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++
+CFLAGS  += --target=aarch64-none-linux-android21 --gcc-toolchain=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64 --sysroot=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot -fdata-sections -ffunction-sections 
+LDFLAGS := --target=aarch64-none-linux-android21 --gcc-toolchain=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64 --sysroot=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot -Wl,--gc-sections 
+LDFLAGS += -L$(MSDIR) -l$(MSLIB) -pthread -llog -latomic -lm  
+else
+CFLAGS += -g
+LDFLAGS := -L$(MSDIR) -l$(MSLIB) -lpthread -Wl,-rpath,$(MSDIR)
+endif  
+LD := ${CXX}
+
+
+all:$(APP)
+
+$(APP): $(OBJ) $(MSDIR)/lib$(MSLIB).so
+	@mkdir -p bin
+	$(LD) $(OBJ) $(LDFLAGS) -o $@
+
+clean:
+	rm -rf src/*.o bin/ 
+
+
+mrproper: 
+	rm -rf dataset package* msl src/*.o bin/ model/*.mindir model/*.ms 
+
+%.o:%.cc
+	$(CXX) $(CFLAGS) -c $< -o $@

mindspore/lite/examples/transfer_learning/README.md

@@ -0,0 +1,150 @@
+# Content
+
+<!-- TOC -->
+
+- [Overview](#overview)
+- [Model Architecture](#model-architecture)
+- [Dataset](#dataset)
+- [Environment Requirements](#environment-requirements)
+- [Quick Start](#quick-start)
+- [Script Detailed Description](#script-detailed-description)
+
+<!-- /TOC -->
+
+# Overview
+
+This folder holds an example code for on-Device Transfer learning. Part of the code runs on a server using MindSpore infrastructure, another part uses MindSpore Lite conversion utility, and the last part is the actual training of the model on some android-based device.
+
+# Transfer Learning Scheme
+
+In this example of transfer learning we are using a fixed backbone that holds a pretrained [efficientNet](https://arxiv.org/abs/1905.11946) and an adjustable head which is a simple Dense layer. During the training only the head will be updating it's weights. Such training scheme is very efficient in computation power and therefore very relevant for Training-on-Device scenarios.
+
+# Dataset
+
+In this example we use a subset of the [Places dataset](http://places2.csail.mit.edu/) of scenes.
+The whole dataset is composed of high resolution as well as small images and sums up to more than 100Gb.
+For this demo we will use only the [validation data of small images](http://places2.csail.mit.edu/download.html) which is approximately 500Mb.
+
+- Dataset size：501M，36,500 224*224 images in 365 classes
+- Dataiset format：jpg files
+    - Note：In the current release, data is customely loaded using a proprietary DataSet class (provided in dataset.cc). In the upcoming releases loading will be done using MindSpore MindData infrastructure. In order to fit the data to the model it will be preprocessed using [ImageMagick convert tool](https://imagemagick.org/), namely croping and converting to bmp format.
+    - Note: Only 10 classes out of the 365 will be used in this demo
+    - Note: 60% of the data will be used for training and 20% will be used for testing and the remaining 20% for validation
+
+- The original dataset directory structure is as follows:
+
+```text
+places
+├── val_256
+│   ├── Places365_val_00000001.jpg
+│   ├── Places365_val_00000002.jpg
+│   ├── ...
+│   ├── Places365_val_00036499.jpg
+│   └── Places365_val_00036500.jpg
+```
+
+# Environment Requirements
+
+- Server side
+    - [MindSpore Framework](https://www.mindspore.cn/install/en) - it is recommended to install a docker image
+    - [MindSpore ToD Framework](https://www.mindspore.cn/tutorial/tod/en/use/prparation.html)
+    - [Android NDK r20b](https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip)
+    - [Android SDK](https://developer.android.com/studio?hl=zh-cn#cmdline-tools)
+    - [ImageMagick convert tool](https://imagemagick.org/)
+- A connected Android device
+
+# Quick Start
+
+After installing all the above mentioned, the script in the home directory could be run with the following arguments:
+
+```bash
+sh ./prepare_and_run.sh -D DATASET_PATH [-d MINDSPORE_DOCKER] [-r RELEASE.tar.gz] [-t arm64|x86]
+```
+
+where:
+
+- DATASET_PATH is the path to the [dataset](#dataset),
+- MINDSPORE_DOCKER is the image name of the docker that runs [MindSpore](#environment-requirements). If not provided MindSpore will be run locally
+- REALEASE.tar.gz is a pointer to the MindSpore ToD release tar ball. If not provided, the script will attempt to find MindSpore ToD compilation output
+- target is defaulted to arm64, i.e., on-device. If x86 is provided, the demo will be run locally. Note that infrastructure is not optimized for device
+
+# Script Detailed Description
+
+The provided `prepare_and_run.sh` script is performing the followings:
+
+- Prepare the trainable transsfer learning model in a `.ms` format
+- Prepare the folder that should be pushed into the device
+- Copy this folder into the device and run the scripts on the device
+
+See how to run the script and paramaters definitions in the [Quick Start Section](#quick-start)
+
+## Preparing the model
+
+Within the model folder a `prepare_model.sh` script uses MindSpore infrastructure to export the model into a `.mindir` file. The user can specify a docker image on which MindSpore is installed. Otherwise, the pyhton script will be run locally.
+The script then converts the `.mindir` to a `.ms` format using the MindSpore ToD converter.
+The script accepts a tar ball where the converter resides. Otherwise, the script will attempt to find the converter in the MindSpore ToD build output directory.
+
+## Preparing the Folder
+
+The `transfer_learning_tod.ms` model file is then copied into the `package` folder as well as scripts, the MindSpore ToD library and a subset of the Places dataset. This dataset undergoes preprocessing on the server prior to the packaging.
+Finally, the code (in src) is compiled for the target and the binary is copied into the `package` folder.
+
+### Running the code on the device
+
+To run the code on the device the script first uses `adb` tool to push the `package` folder into the device.
+It first runs evaluation on the untrained model (to check the accuracy of the untrained model)
+It then runs training (which takes some time) and finally it runs evaluation of the trained model using the test data.
+
+# Folder Directory tree
+
+```text
+transfer_learning/
+├── Makefile                          # Makefile of src code
+├── model
+│   ├── effnet.py                     # Python implementation of efficientNet
+│   ├── transfer_learning_export.py   # Python script that exports the LeNet model to .mindir
+│   ├── prepare_model.sh              # script that export model (using docker) then converts it
+│   └── train_utils.py                # utility function used during the export
+├── prepare_and_run.sh                # main script that creates model, compiles it and send to device for running
+├── prepare_dataset.sh                # prepares the Places dataset (crop/convert/organizing folders)
+├── README.md                         # this manual
+├── scripts
+│   ├── eval.sh                       # script that load the train model and evaluates its accuracy
+│   ├── eval_untrained.sh             # script that load the untrained model and evaluates its accuracy
+│   ├── places365_val.txt             # association of images to classes withiin the Places 365 dataset
+│   └── train.sh                      # script that load the initial model and trains it
+├── src
+│   ├── dataset.cc        # dataset handler
+│   ├── dataset.h         # dataset class header
+│   ├── net_runner.cc     # program that runs training/evaluation of models
+│   └── net_runner.h      # net_runner header
+```
+
+When the `prepare_and_run.sh` script is run, the following folder is prepared. It is pushed to the device and then training runs
+
+```text
+package-arm64/
+├── bin
+│   └── net_runner                     # the executable that performs the training/evaluation
+├── dataset
+│   ├── 0                              # folder containing images 0-99 belonging to 0'th class
+│   │   ├── 0.bmp
+│   │   ├── 1.bmp
+│   │   ├── ....
+│   │   ├── 98.bmp
+│   │   └── 99.bmp
+│   ├── ...                            # folders containing images 0-99 belonging to 1'st-8'th classes
+│   ├── 9                              # folder containing images 0-99 belonging to 9'th class
+│   │   ├── 0.bmp
+│   │   ├── 1.bmp
+│   │   ├── ....
+│   │   ├── 98.bmp
+│   │   └── 99.bmp
+├── lib
+│   └── libmindspore-lite.so           # MindSpore Lite library
+├── model
+│   └── transfer_learning_tod.ms       # model to train
+├── eval.sh                            # script that load the train model and evaluates its accuracy
+├── eval_untrained.sh                  # script that load the untrain model and evaluates its accuracy
+└── train.sh                           # script that load the initial model and train it
+```

mindspore/lite/examples/transfer_learning/model/effnet.py

@@ -0,0 +1,317 @@
+"""EffNet model define"""
+import numpy as np
+import mindspore.nn as nn
+from mindspore.ops import operations as P
+from mindspore.common.initializer import TruncatedNormal
+from mindspore import Tensor
+
+
+__all__ = ['effnet']
+
+
+def weight_variable():
+    """weight initial"""
+    return TruncatedNormal(0.02)
+
+
+def _make_divisible(v, divisor, min_value=None):
+    """
+    This function is taken from the original tf repo.
+    It ensures that all layers have a channel number that is divisible by 8
+    It can be seen here:
+    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py
+    :param v:
+    :param divisor:
+    iparam min_value:
+    :return:
+    """
+    if min_value is None:
+        min_value = divisor
+    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)
+    # Make sure that round down does not go down by more than 10%.
+    if new_v < 0.9 * v:
+        new_v += divisor
+    return new_v
+
+
+class Swish(nn.Cell):
+    def __init__(self):
+        super().__init__(Swish)
+        self.sigmoid = nn.Sigmoid()
+
+    def construct(self, x):
+        s = self.sigmoid(x)
+        m = x*s
+        return m
+
+class AdaptiveAvgPool(nn.Cell):
+    def __init__(self, output_size=None):
+        super().__init__(AdaptiveAvgPool)
+        self.mean = P.ReduceMean(keep_dims=True)
+        self.output_size = output_size
+
+    def construct(self, x):
+        return self.mean(x, (2, 3))
+
+class SELayer(nn.Cell):
+    """
+    SELayer
+    """
+    def __init__(self, channel, reduction=4):
+        super().__init__(SELayer)
+        reduced_chs = _make_divisible(channel/reduction, 1)
+        self.avg_pool = AdaptiveAvgPool(output_size=(1, 1))
+        weight = weight_variable()
+        self.conv_reduce = nn.Conv2d(
+            in_channels=channel, out_channels=reduced_chs, kernel_size=1, has_bias=True, weight_init=weight)
+        self.act1 = Swish()
+        self.conv_expand = nn.Conv2d(
+            in_channels=reduced_chs, out_channels=channel, kernel_size=1, has_bias=True)
+        self.act2 = nn.Sigmoid()
+
+    def construct(self, x):
+        o = self.avg_pool(x)
+        o = self.conv_reduce(o)
+        o = self.act1(o)
+        o = self.conv_expand(o)
+        o = self.act2(o)
+        return x * o
+
+class DepthwiseSeparableConv(nn.Cell):
+    """
+    DepthwiseSeparableConv
+    """
+    def __init__(self, in_chs, out_chs, dw_kernel_size=3,
+                 stride=1, noskip=False, se_ratio=0.0, drop_connect_rate=0.0):
+        super().__init__(DepthwiseSeparableConv)
+        assert stride in [1, 2]
+        self.has_residual = (stride == 1 and in_chs == out_chs) and not noskip
+        self.drop_connect_rate = drop_connect_rate
+
+        self.conv_dw = nn.Conv2d(in_channels=in_chs, out_channels=in_chs, kernel_size=dw_kernel_size,
+                                 stride=stride, pad_mode="pad", padding=1, has_bias=False, group=in_chs)
+        self.bn1 = nn.BatchNorm2d(in_chs, eps=0.001)
+        self.act1 = Swish()
+
+        if se_ratio is not None and se_ratio > 0.:
+            self.se = SELayer(in_chs, reduction=se_ratio)
+        else:
+            print("ERRRRRORRRR -- not prepared for this one\n")
+
+        self.conv_pw = nn.Conv2d(
+            in_channels=in_chs, out_channels=out_chs, kernel_size=1, stride=stride, has_bias=False)
+        self.bn2 = nn.BatchNorm2d(out_chs, eps=0.001)
+
+    def construct(self, x):
+        """
+        construct
+        """
+        residual = x
+        x = self.conv_dw(x)
+        x = self.bn1(x)
+        x = self.act1(x)
+        x = self.se(x)
+        x = self.conv_pw(x)
+        x = self.bn2(x)
+        if self.has_residual:
+            x += residual
+        return x
+
+def conv_3x3_bn(inp, oup, stride):
+    weight = weight_variable()
+    return nn.SequentialCell([
+        nn.Conv2d(in_channels=inp, out_channels=oup, kernel_size=3, stride=stride,
+                  padding=1, weight_init=weight, has_bias=False, pad_mode='pad'),
+        nn.BatchNorm2d(oup, eps=0.001),  # , momentum=0.1),
+        nn.HSwish()])
+
+def conv_1x1_bn(inp, oup):
+    weight = weight_variable()
+    return nn.SequentialCell([
+        nn.Conv2d(in_channels=inp, out_channels=oup, kernel_size=1,
+                  stride=1, padding=0, weight_init=weight, has_bias=False),
+        nn.BatchNorm2d(oup, eps=0.001),
+        nn.HSwish()])
+
+class InvertedResidual(nn.Cell):
+    """
+    InvertedResidual
+    """
+    def __init__(self, in_chs, out_chs, kernel_size, stride, padding, expansion, se_ratio):
+        super().__init__(InvertedResidual)
+        assert stride in [1, 2]
+        mid_chs: int = _make_divisible(in_chs * expansion, 1)
+        self.has_residual = (in_chs == out_chs and stride == 1)
+        self.drop_connect_rate = 0
+        self.conv_pw = nn.Conv2d(
+            in_channels=in_chs, out_channels=mid_chs, kernel_size=1, stride=1, has_bias=False)
+        self.bn1 = nn.BatchNorm2d(mid_chs, eps=0.001)  # ,momentum=0.1)
+        self.act1 = Swish()
+        if stride > 1:
+            self.conv_dw = nn.Conv2d(in_channels=mid_chs, out_channels=mid_chs, kernel_size=kernel_size,
+                                     stride=stride, padding=padding, has_bias=False, group=mid_chs, pad_mode='same')
+        else:
+            self.conv_dw = nn.Conv2d(in_channels=mid_chs, out_channels=mid_chs, kernel_size=kernel_size,
+                                     stride=stride, padding=padding, has_bias=False, group=mid_chs, pad_mode='pad')
+        self.bn2 = nn.BatchNorm2d(mid_chs, eps=0.001)  # ,momentum=0.1)
+        self.act2 = Swish()
+
+        # Squeeze-and-excitation
+        if se_ratio is not None and se_ratio > 0.:
+            self.se = SELayer(mid_chs, reduction=se_ratio)
+        else:
+            print("ERRRRRORRRR -- not prepared for this one\n")
+
+        # Point-wise linear projection
+        self.conv_pwl = nn.Conv2d(
+            in_channels=mid_chs, out_channels=out_chs, kernel_size=1, stride=1, has_bias=False)
+        self.bn3 = nn.BatchNorm2d(out_chs, eps=0.001)  # ,momentum=0.1)
+
+    def construct(self, x):
+        """
+        construct
+        """
+        residual = x
+
+        # Point-wise expansion
+        x = self.conv_pw(x)
+        x = self.bn1(x)
+        x = self.act1(x)
+
+        # Depth-wise convolution
+        x = self.conv_dw(x)
+        x = self.bn2(x)
+        x = self.act2(x)
+
+        # Squeeze-and-excitation
+        x = self.se(x)
+
+        # Point-wise linear projection
+        x = self.conv_pwl(x)
+        x = self.bn3(x)
+
+        if self.has_residual:
+            x += residual
+        return x
+
+class EfficientNet(nn.Cell):
+    """
+    EfficientNet
+    """
+    def __init__(self, cfgs, num_classes=1000):
+        super().__init__(EfficientNet)
+        # setting of inverted residual blocks
+        self.cfgs = cfgs
+        stem_size = 32
+        self.num_classes_ = num_classes
+        self.num_features_ = 1280
+
+        self.conv_stem = nn.Conv2d(
+            in_channels=3, out_channels=stem_size, kernel_size=3, stride=2, has_bias=False)
+
+        self.bn1 = nn.BatchNorm2d(stem_size, eps=0.001)  # momentum=0.1)
+        self.act1 = Swish()
+        in_chs = stem_size
+
+        layers = [nn.SequentialCell([DepthwiseSeparableConv(in_chs, 16, 3, 1, se_ratio=4)]),
+
+                  nn.SequentialCell([InvertedResidual(16, 24, 3, 2, 0, 6, se_ratio=24),
+                                     InvertedResidual(24, 24, 3, 1, 1, 6, se_ratio=24)]),
+
+                  nn.SequentialCell([InvertedResidual(24, 40, 5, 2, 0, 6, se_ratio=24),
+                                     InvertedResidual(40, 40, 5, 1, 2, 6, se_ratio=24)]),
+
+                  nn.SequentialCell([InvertedResidual(40, 80, 3, 2, 0, 6, se_ratio=24),
+                                     InvertedResidual(
+                                         80, 80, 3, 1, 1, 6, se_ratio=24),
+                                     InvertedResidual(80, 80, 3, 1, 1, 6, se_ratio=24)]),
+
+                  nn.SequentialCell([InvertedResidual(80, 112, 5, 1, 2, 6, se_ratio=24),
+                                     InvertedResidual(
+                                         112, 112, 5, 1, 2, 6, se_ratio=24),
+                                     InvertedResidual(112, 112, 5, 1, 2, 6, se_ratio=24)]),
+
+                  nn.SequentialCell([InvertedResidual(112, 192, 5, 2, 0, 6, se_ratio=24),
+                                     InvertedResidual(
+                                         192, 192, 5, 1, 2, 6, se_ratio=24),
+                                     InvertedResidual(
+                                         192, 192, 5, 1, 2, 6, se_ratio=24),
+                                     InvertedResidual(192, 192, 5, 1, 2, 6, se_ratio=24)]),
+
+                  nn.SequentialCell(
+                      [InvertedResidual(192, 320, 3, 1, 1, 6, se_ratio=24)])
+                  ]
+        self.blocks = nn.SequentialCell(layers)
+
+        self.conv_head = nn.Conv2d(
+            in_channels=320, out_channels=self.num_features_, kernel_size=1)
+        self.bn2 = nn.BatchNorm2d(self.num_features_, eps=0.001)
+        self.act2 = Swish()
+        self.global_pool = AdaptiveAvgPool(output_size=(1, 1))
+        self.classifier = nn.Dense(self.num_features_, num_classes)
+
+        self._initialize_weights()
+
+    def construct(self, x):
+        """
+        construct
+        """
+        x = self.conv_stem(x)
+        x = self.bn1(x)
+        x = self.act1(x)
+        x = self.blocks(x)
+        x = self.conv_head(x)
+        x = self.bn2(x)
+        x = self.act2(x)
+        x = self.global_pool(x)
+        x = P.Reshape()(x, (-1, self.num_features_))
+        x = self.classifier(x)
+        return x
+
+    def _initialize_weights(self):
+        """
+        _initialize_weights
+        """
+        def init_linear_weight(m):
+            m.weight.set_data(Tensor(np.random.normal(
+                0, 0.01, m.weight.data.shape).astype("float32")))
+            if m.bias is not None:
+                m.bias.set_data(
+                    Tensor(np.zeros(m.bias.data.shape, dtype="float32")))
+
+        for m in self.cells():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.set_data(Tensor(np.random.normal(0, np.sqrt(2. / n),
+                                                          m.weight.data.shape).astype("float32")))
+                if m.bias is not None:
+                    m.bias.data.zero_()
+                m.weight.requires_grad = True
+            elif isinstance(m, nn.BatchNorm2d):
+                m.gamma.set_data(
+                    Tensor(np.ones(m.gamma.data.shape, dtype="float32")))
+                m.beta.set_data(
+                    Tensor(np.zeros(m.beta.data.shape, dtype="float32")))
+            elif isinstance(m, nn.Dense):
+                init_linear_weight(m)
+
+def effnet(**kwargs):
+    """
+    Constructs a EfficientNet model
+    """
+    cfgs = [
+        # k, t, c, SE, HS, s
+        [3, 1, 16, 1, 0, 2],
+        [3, 4.5, 24, 0, 0, 2],
+        [3, 3.67, 24, 0, 0, 1],
+        [5, 4, 40, 1, 1, 2],
+        [5, 6, 40, 1, 1, 1],
+        [5, 6, 40, 1, 1, 1],
+        [5, 3, 48, 1, 1, 1],
+        [5, 3, 48, 1, 1, 1],
+        [5, 6, 96, 1, 1, 2],
+        [5, 6, 96, 1, 1, 1],
+        [5, 6, 96, 1, 1, 1],
+    ]
+
+    return EfficientNet(cfgs, **kwargs)

mindspore/lite/examples/transfer_learning/model/prepare_model.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ ! -f "efficient_net_b0.ckpt" ]; then
+  echo "Pretrained model weights are missing, downloading efficient_net_b0.ckpt"
+  wget https://download.mindspore.cn/model_zoo/official/lite/efficient_net/efficient_net_b0.ckpt
+fi
+
+echo "============Exporting=========="
+if [ -n "$1" ]; then
+  DOCKER_IMG=$1
+  docker run -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER --privileged=true ${DOCKER_IMG} /bin/bash -c "python transfer_learning_export.py; chmod 444 transfer_learning_tod.mindir; rm -rf __pycache__"
+else
+  echo "MindSpore docker was not provided, attempting to run locally"
+  python transfer_learning_export.py
+fi
+
+CONVERTER="../../../build/tools/converter/converter_lite"
+if [ ! -f "$CONVERTER" ]; then
+  if ! command -v converter_lite &> /dev/null
+  then
+    tar -xzf ../../../../../output/mindspore-lite-*-converter-ubuntu-train.tar.gz --strip-components 2 --wildcards --no-anchored converter_lite libmindspore_gvar.so
+    if [ -f ./converter_lite ]; then
+      CONVERTER=./converter_lite
+    else
+      echo "converter_lite could not be found in MindSpore build directory nor in system path"
+      exit 1
+    fi
+  else
+    CONVERTER=converter_lite
+  fi
+fi
+
+echo "============Converting========="
+LD_LIBRARY_PATH=./ $CONVERTER --fmk=MINDIR --trainModel=true --modelFile=transfer_learning_tod.mindir --outputFile=transfer_learning_tod

mindspore/lite/examples/transfer_learning/model/train_utils.py

@@ -0,0 +1,34 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""train_utils."""
+
+import mindspore.nn as nn
+from mindspore.common.parameter import ParameterTuple
+
+def TrainWrap(net, loss_fn=None, optimizer=None, weights=None):
+    """
+    TrainWrap
+    """
+    if loss_fn is None:
+        loss_fn = nn.SoftmaxCrossEntropyWithLogits(reduction='mean')
+    loss_net = nn.WithLossCell(net, loss_fn)
+    loss_net.set_train()
+    if weights is None:
+        weights = ParameterTuple(net.trainable_params())
+    if optimizer is None:
+        optimizer = nn.Adam(weights, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8,
+                            use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0)
+    train_net = nn.TrainOneStepCell(loss_net, optimizer)
+    return train_net

mindspore/lite/examples/transfer_learning/model/transfer_learning_export.py

@@ -0,0 +1,63 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""transfer_learning_export."""
+
+import mindspore as M
+from mindspore.nn import Cell
+from mindspore.train.serialization import load_checkpoint
+from mindspore.common.parameter import ParameterTuple
+from mindspore.train.serialization import export
+from effnet import effnet
+import numpy as np
+from train_utils import TrainWrap
+
+
+class TransferNet(Cell):
+    def __init__(self, backbone, head):
+        super().__init__(TransferNet)
+        self.backbone = backbone
+        self.head = head
+
+    def construct(self, x):
+        x = self.backbone(x)
+        x = self.head(x)
+        return x
+
+
+BACKBONE = effnet(num_classes=1000)
+load_checkpoint("efficient_net_b0.ckpt", BACKBONE)
+HEAD = M.nn.Dense(1000, 10)
+HEAD.weight.set_data(M.Tensor(np.random.normal(
+    0, 0.1, HEAD.weight.data.shape).astype("float32")))
+HEAD.bias.set_data(M.Tensor(np.zeros(HEAD.bias.data.shape, dtype="float32")))
+
+n = TransferNet(BACKBONE, HEAD)
+
+trainable_weights_list = []
+trainable_weights_list.extend(n.head.trainable_params())
+trainable_weights = ParameterTuple(trainable_weights_list)
+
+M.context.set_context(mode=M.context.PYNATIVE_MODE,
+                      device_target="GPU", save_graphs=False)
+BATCH_SIZE = 32
+X = M.Tensor(np.ones((BATCH_SIZE, 3, 224, 224)), M.float32)
+label = M.Tensor(np.zeros([BATCH_SIZE, 10]).astype(np.float32))
+
+sgd = M.nn.SGD(trainable_weights, learning_rate=0.01, momentum=0.9,
+               dampening=0.01, weight_decay=0.0, nesterov=False, loss_scale=1.0)
+net = TrainWrap(n, optimizer=sgd, weights=trainable_weights)
+export(net, X, label, file_name="transfer_learning_tod.mindir", file_format='MINDIR')
+
+print("Exported")

mindspore/lite/examples/transfer_learning/prepare_and_run.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+display_usage()
+{
+  echo -e "\nUsage: prepare_and_run.sh -D dataset_path [-d mindspore_docker] [-r release.tar.gz] [-t arm64|x86]\n"
+}
+
+checkopts()
+{
+  TARGET="arm64"
+  DOCKER=""
+  PLACES_DATA_PATH=""
+  while getopts 'D:d:r:t:' opt
+  do
+    OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
+    case "${opt}" in
+      D)
+        PLACES_DATA_PATH=$OPTARG
+        ;;
+      d)
+        DOCKER=$OPTARG
+        ;;
+      t)
+        if [ "$OPTARG" == "arm64" ] || [ "$OPTARG" == "x86" ]; then
+          TARGET=$OPTARG
+        else
+          echo "No such target " $OPTARG
+          display_usage
+          exit 1
+        fi
+        ;;
+      r)
+        TARBALL=$OPTARG
+        ;;
+      *)
+        echo "Unknown option ${opt}!"
+        display_usage
+        exit 1
+    esac
+  done
+}
+checkopts "$@"
+if [ "$PLACES_DATA_PATH" == "" ]; then
+  echo "Places Dataset directory path was not provided"
+  display_usage
+  exit 1
+fi
+
+if [ "$TARBALL" == "" ]; then
+  file=$(ls ../../../../output/mindspore-lite-*-runtime-${TARGET}-cpu-train.tar.gz)
+  if [ -f ${file} ]; then
+    TARBALL=${file}
+  else
+    echo "release.tar.gz was not found"
+    display_usage
+    exit 1
+  fi
+fi
+
+# Prepare the model
+cd model/ || exit 1
+rm -f *.ms
+./prepare_model.sh $DOCKER || exit 1
+cd ../
+
+# Copy the .ms model to the package folder
+
+PACKAGE=package-${TARGET}
+
+rm -rf ${PACKAGE}
+mkdir -p ${PACKAGE}/model
+cp model/*.ms ${PACKAGE}/model
+
+# Copy the running script to the package
+cp scripts/*.sh ${PACKAGE}/
+
+# Copy the shared MindSpore ToD library
+tar -xzf ${TARBALL} --wildcards --no-anchored libmindspore-lite.so
+tar -xzf ${TARBALL} --wildcards --no-anchored include
+mv mindspore-*/lib ${PACKAGE}/
+rm -rf msl
+mkdir msl
+mv mindspore-*/* msl/
+rm -rf mindspore-*
+
+# Convert the dataset into the package
+./prepare_dataset.sh ${PLACES_DATA_PATH}
+cp -r dataset ${PACKAGE}
+
+echo "==========Compiling============"
+make TARGET=${TARGET}
+ 
+# Copy the executable to the package
+mv bin ${PACKAGE}/ || exit 1
+
+if [ "${TARGET}" == "arm64" ]; then
+  echo "=======Pushing to device======="
+  adb push ${PACKAGE} /data/local/tmp/
+
+  echo "==Evaluating Untrained Model==="
+  adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh eval_untrained.sh"
+
+  echo "========Training on Device====="
+  adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh train.sh"
+
+  echo
+  echo "===Evaluating trained Model====="
+  adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh eval.sh"
+  echo
+else
+  cd ${PACKAGE} || exit 1
+  echo "==Evaluating Untrained Model==="
+  ./eval_untrained.sh
+
+  echo "======Training Locally========="
+  ./train.sh
+
+  echo "===Evaluating trained Model====="
+  ./eval.sh
+  cd ..
+fi
+

mindspore/lite/examples/transfer_learning/prepare_dataset.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo "=======Preparing Dataset======="
+[ -d "dataset" ] && echo "dataset was already created" && exit
+echo "Preparing dataset" 
+PLACES_DATA_PATH=$1
+class_id=0
+classes=("4" "98" "6" "7" "10" "15" "17" "70" "26" "30")
+for class in "${classes[@]}"; do
+  mkdir -p dataset/$class_id
+  i=0
+  cat scripts/places365_val.txt | grep -w ${class} | awk '{print $1}' | while read line
+  do 
+    echo converting ${PLACES_DATA_PATH}/val_256/$line to bmp
+    convert -colorspace RGB -gravity center -crop '224x224+0+0' ${PLACES_DATA_PATH}/val_256/$line dataset/$class_id/$i.bmp; 
+    i=$(($i+1)); 
+  done
+  class_id=$(($class_id+1))
+done

mindspore/lite/examples/transfer_learning/scripts/eval.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/transfer_learning_tod_trained.ms -e 0 -d dataset 

mindspore/lite/examples/transfer_learning/scripts/eval_untrained.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/transfer_learning_tod.ms -e 0 -d dataset 

mindspore/lite/examples/transfer_learning/scripts/train.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/transfer_learning_tod.ms -e 60 -d dataset -s 20

mindspore/lite/examples/transfer_learning/src/dataset.cc

@@ -0,0 +1,209 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/dataset.h"
+#include <assert.h>
+#include <dirent.h>
+#include <arpa/inet.h>
+#include <map>
+#include <iostream>
+#include <fstream>
+#include <memory>
+#include <filesystem>
+
+#pragma pack(push, 1)
+
+typedef struct {  // Total: 54 bytes
+  uint16_t type;  // Magic identifier: 0x4d42
+  uint32_t size;
+  uint16_t reserved1;
+  uint16_t reserved2;
+  uint32_t offset;
+  uint32_t dib_header_size;
+  int32_t width;
+  int32_t height;
+  uint16_t channels;
+  uint16_t bits_per_pixel;
+  uint32_t compression;
+  uint32_t image_size_bytes;
+  int32_t x_resolution_ppm;
+  int32_t y_resolution_ppm;
+  uint32_t num_colors;
+  uint32_t important_colors;
+} bmp_header;
+
+#pragma pack(pop)
+
+float CH_MEAN[3] = {0.485, 0.456, 0.406};
+float CH_STD[3] = {0.229, 0.224, 0.225};
+
+using LabelId = std::map<std::string, int>;
+
+static char *ReadBitmapFile(const std::string &filename, size_t *size) {
+  assert(size != nullptr);
+  *size = 0;
+  bmp_header bitmap_header;
+  std::ifstream ifs(filename);
+  if (!ifs.good() || !ifs.is_open()) {
+    std::cerr << "file: " << filename << " does not exist or failed to open";
+    return nullptr;
+  }
+
+  ifs.read(reinterpret_cast<char *>(&bitmap_header), sizeof(bmp_header));
+  if (bitmap_header.type != 0x4D42) {
+    std::cerr << "file: " << filename << " magic number does not match BMP";
+    ifs.close();
+    return nullptr;
+  }
+
+  ifs.seekg(bitmap_header.offset, std::ios::beg);
+
+  unsigned char *bmp_image = reinterpret_cast<unsigned char *>(malloc(bitmap_header.image_size_bytes));
+  if (!bmp_image) {
+    ifs.close();
+    return nullptr;
+  }
+
+  ifs.read(reinterpret_cast<char *>(bmp_image), bitmap_header.image_size_bytes);
+
+  size_t buffer_size = bitmap_header.width * bitmap_header.height * 3;
+  float *hwc_bin_image = new (std::nothrow) float[buffer_size];
+  if (!hwc_bin_image) {
+    free(bmp_image);
+    ifs.close();
+    return nullptr;
+  }
+
+  // swap the R and B values to get RGB (bitmap is BGR)
+  // swap columns (in BMP, first pixel is lower left one...)
+  const size_t channels = 3;
+  const size_t hStride = channels * bitmap_header.width;
+  const size_t height = bitmap_header.height;
+
+  for (int h = 0; h < bitmap_header.height; h++) {
+    for (int w = 0; w < bitmap_header.width; w++) {
+      hwc_bin_image[h * hStride + w * channels + 0] =
+        (((static_cast<float>(bmp_image[(height - h - 1) * hStride + w * channels + 2])) / 255.0) - CH_MEAN[0]) /
+        CH_STD[0];
+      hwc_bin_image[h * hStride + w * channels + 1] =
+        (((static_cast<float>(bmp_image[(height - h - 1) * hStride + w * channels + 1])) / 255.0) - CH_MEAN[1]) /
+        CH_STD[1];
+      hwc_bin_image[h * hStride + w * channels + 2] =
+        (((static_cast<float>(bmp_image[(height - h - 1) * hStride + w * channels + 0])) / 255.0) - CH_MEAN[2]) /
+        CH_STD[2];
+    }
+  }
+
+  *size = buffer_size * sizeof(float);
+  free(bmp_image);
+  ifs.close();
+  char *ret_buf = reinterpret_cast<char *>(hwc_bin_image);
+  return ret_buf;
+}
+
+char *ReadFile(const std::string &file, size_t *size) {
+  assert(size != nullptr);
+  std::string realPath(file);
+  std::ifstream ifs(realPath);
+  if (!ifs.good()) {
+    std::cerr << "file: " << realPath << " does not exist";
+    return nullptr;
+  }
+
+  if (!ifs.is_open()) {
+    std::cerr << "file: " << realPath << " open failed";
+    return nullptr;
+  }
+
+  ifs.seekg(0, std::ios::end);
+  *size = ifs.tellg();
+  std::unique_ptr<char[]> buf(new (std::nothrow) char[*size]);
+  if (buf == nullptr) {
+    std::cerr << "malloc buf failed, file: " << realPath;
+    ifs.close();
+    return nullptr;
+  }
+
+  ifs.seekg(0, std::ios::beg);
+  ifs.read(buf.get(), *size);
+  ifs.close();
+
+  return buf.release();
+}
+
+DataSet::~DataSet() {
+  for (auto itr = train_data_.begin(); itr != train_data_.end(); ++itr) {
+    auto ptr = std::get<0>(*itr);
+    delete[] ptr;
+  }
+  for (auto itr = test_data_.begin(); itr != test_data_.end(); ++itr) {
+    auto ptr = std::get<0>(*itr);
+    delete[] ptr;
+  }
+  for (auto itr = val_data_.begin(); itr != val_data_.end(); ++itr) {
+    auto ptr = std::get<0>(*itr);
+    delete[] ptr;
+  }
+}
+
+int DataSet::Init(const std::string &data_base_directory, database_type type) {
+  InitializeBMPFoldersDatabase(data_base_directory);
+  return 0;
+}
+
+void DataSet::InitializeBMPFoldersDatabase(std::string dpath) {
+  size_t file_size = 0;
+  const int ratio = 5;
+  auto vec = ReadDir(dpath);
+  int running_index = 1;
+  for (const auto ft : vec) {
+    int label;
+    std::string file_name;
+    std::tie(label, file_name) = ft;
+    char *data = ReadBitmapFile(file_name, &file_size);
+    DataLabelTuple data_entry = std::make_tuple(data, label);
+    if ((expected_data_size_ == 0) || (file_size == expected_data_size_)) {
+      if (running_index % ratio == 0) {
+        val_data_.push_back(data_entry);
+      } else if (running_index % ratio == 1) {
+        test_data_.push_back(data_entry);
+      } else {
+        train_data_.push_back(data_entry);
+      }
+      running_index++;
+    }
+  }
+}
+
+std::vector<FileTuple> DataSet::ReadDir(const std::string dpath) {
+  std::vector<FileTuple> vec;
+  struct dirent *entry = nullptr;
+  num_of_classes_ = 10;
+  for (int class_id = 0; class_id < num_of_classes_; class_id++) {
+    std::string dirname = dpath + "/" + std::to_string(class_id);
+    DIR *dp = opendir(dirname.c_str());
+    if (dp != nullptr) {
+      while ((entry = readdir(dp))) {
+        std::string filename = dirname + "/" + entry->d_name;
+        if (filename.find(".bmp") != std::string::npos) {
+          FileTuple ft = make_tuple(class_id, filename);
+          vec.push_back(ft);
+        }
+      }
+    }
+  }
+  return vec;
+}

mindspore/lite/examples/transfer_learning/src/dataset.h

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_EXAMPLES_TRANSFER_LEARNING_SRC_DATASET_H_
+#define MINDSPORE_LITE_EXAMPLES_TRANSFER_LEARNING_SRC_DATASET_H_
+
+#include <tuple>
+#include <string>
+#include <vector>
+
+using DataLabelTuple = std::tuple<char *, int>;
+using FileTuple = std::tuple<int, std::string>;
+
+enum database_type { DS_CIFAR10_BINARY = 0, DS_MNIST_BINARY, DS_OTHER };
+
+char *ReadFile(const std::string &file, size_t *size);  // utility function
+
+class DataSet {
+ public:
+  DataSet() {}
+  ~DataSet();
+
+  int Init(const std::string &data_base_directory, database_type type = DS_OTHER);
+
+  const std::vector<DataLabelTuple> &train_data() const { return train_data_; }
+  const std::vector<DataLabelTuple> &test_data() const { return test_data_; }
+  const std::vector<DataLabelTuple> &val_data() const { return val_data_; }
+  unsigned int num_of_classes() { return num_of_classes_; }
+  void set_expected_data_size(unsigned int expected_data_size) { expected_data_size_ = expected_data_size; }
+  unsigned int expected_data_size() { return expected_data_size_; }
+
+ private:
+  std::vector<FileTuple> ReadDir(const std::string dpath);
+  void InitializeBMPFoldersDatabase(std::string dpath);
+
+  std::vector<DataLabelTuple> train_data_;
+  std::vector<DataLabelTuple> test_data_;
+  std::vector<DataLabelTuple> val_data_;
+  unsigned int num_of_classes_ = 0;
+  unsigned int expected_data_size_ = 0;
+};
+
+#endif  // MINDSPORE_LITE_EXAMPLES_TRANSFER_LEARNING_SRC_DATASET_H_

mindspore/lite/examples/transfer_learning/src/net_runner.cc

@@ -0,0 +1,259 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/net_runner.h"
+#include <math.h>
+#include <getopt.h>
+#include <iostream>
+#include <fstream>
+#include "include/context.h"
+
+static unsigned int seed = time(NULL);
+
+// Definition of callback function after forwarding operator.
+bool after_callback(const std::vector<mindspore::tensor::MSTensor *> &after_inputs,
+                    const std::vector<mindspore::tensor::MSTensor *> &after_outputs,
+                    const mindspore::CallBackParam &call_param) {
+  std::cout << call_param.node_name << std::endl;
+  for (size_t i = 0; i < after_inputs.size(); i++) {
+    int num2p = (after_inputs.at(i)->ElementsNum());
+    std::cout << "in" << i << "(" << num2p << "): ";
+    if (num2p > 10) num2p = 10;
+    if (after_inputs.at(i)->data_type() == mindspore::kNumberTypeInt32) {
+      auto d = reinterpret_cast<int *>(after_inputs.at(i)->MutableData());
+      for (int j = 0; j < num2p; j++) {
+        std::cout << d[j] << ", ";
+      }
+    } else {
+      auto d = reinterpret_cast<float *>(after_inputs.at(i)->MutableData());
+      for (int j = 0; j < num2p; j++) {
+        std::cout << d[j] << ", ";
+      }
+    }
+    std::cout << std::endl;
+  }
+  for (size_t i = 0; i < after_outputs.size(); i++) {
+    auto d = reinterpret_cast<float *>(after_outputs.at(i)->MutableData());
+    int num2p = (after_outputs.at(i)->ElementsNum());
+    std::cout << "ou" << i << "(" << num2p << "): ";
+    if (num2p > 10) num2p = 10;
+    for (int j = 0; j < num2p; j++) {
+      std::cout << d[j] << ", ";
+    }
+    std::cout << std::endl;
+  }
+  return true;
+}
+
+NetRunner::~NetRunner() {
+  if (session_ != nullptr) delete session_;
+}
+
+void NetRunner::InitAndFigureInputs() {
+  mindspore::lite::Context context;
+  context.device_list_[0].device_info_.cpu_device_info_.cpu_bind_mode_ = mindspore::lite::NO_BIND;
+  context.thread_num_ = 1;
+
+  session_ = mindspore::session::TrainSession::CreateSession(ms_file_, &context);
+  assert(nullptr != session_);
+
+  auto inputs = session_->GetInputs();
+  assert(inputs.size() > 1);
+  data_index_ = 0;
+  label_index_ = 1;
+  batch_size_ = inputs[data_index_]->shape()[0];
+  data_size_ = inputs[data_index_]->Size() / batch_size_;  // in bytes
+  if (verbose_) {
+    std::cout << "data size: " << data_size_ << std::endl << "batch size: " << batch_size_ << std::endl;
+  }
+}
+
+mindspore::tensor::MSTensor *NetRunner::SearchOutputsForSize(size_t size) const {
+  auto outputs = session_->GetOutputs();
+  for (auto it = outputs.begin(); it != outputs.end(); ++it) {
+    if (it->second->ElementsNum() == size) return it->second;
+  }
+  std::cout << "Model does not have an output tensor with size " << size << std::endl;
+  return nullptr;
+}
+
+std::vector<int> NetRunner::FillInputData(const std::vector<DataLabelTuple> &dataset, int serially) const {
+  std::vector<int> labels_vec;
+  static unsigned int idx = 1;
+  int total_size = dataset.size();
+
+  if (serially == 0) idx = 0;
+  auto inputs = session_->GetInputs();
+  char *input_data = reinterpret_cast<char *>(inputs.at(data_index_)->MutableData());
+  auto labels = reinterpret_cast<float *>(inputs.at(label_index_)->MutableData());
+  assert(total_size > 0);
+  assert(input_data != nullptr);
+  std::fill(labels, labels + inputs.at(label_index_)->ElementsNum(), 0.f);
+  for (int i = 0; i < batch_size_; i++) {
+    if (serially >= 0) {
+      idx = ++idx % total_size;
+    } else {
+      idx = rand_r(&seed) % total_size;
+    }
+    int label = 0;
+    char *data = nullptr;
+    std::tie(data, label) = dataset[idx];
+    memcpy(input_data + i * data_size_, data, data_size_);
+    labels[i * num_of_classes_ + label] = 1.0;  // Model expects labels in onehot representation
+    labels_vec.push_back(label);
+  }
+
+  return labels_vec;
+}
+
+float NetRunner::CalculateAccuracy(const std::vector<DataLabelTuple> &dataset) const {
+  float accuracy = 0.0;
+  int tests = dataset.size() / batch_size_;
+
+  session_->Eval();
+  for (int i = 0; i < tests; i++) {
+    auto labels = FillInputData(dataset, i);
+    session_->RunGraph();
+    auto outputsv = SearchOutputsForSize(batch_size_ * num_of_classes_);
+    assert(outputsv != nullptr);
+    auto scores = reinterpret_cast<float *>(outputsv->MutableData());
+    for (int b = 0; b < batch_size_; b++) {
+      int max_idx = 0;
+      float max_score = scores[num_of_classes_ * b];
+      for (int c = 0; c < num_of_classes_; c++) {
+        if (scores[num_of_classes_ * b + c] > max_score) {
+          max_score = scores[num_of_classes_ * b + c];
+          max_idx = c;
+        }
+      }
+      if (labels[b] == max_idx) accuracy += 1.0;
+    }
+  }
+  session_->Train();
+  accuracy /= static_cast<float>(batch_size_ * tests);
+  return accuracy;
+}
+
+int NetRunner::InitDB() {
+  if (data_size_ != 0) ds_.set_expected_data_size(data_size_);
+  int ret = ds_.Init(data_dir_, DS_OTHER);
+  num_of_classes_ = ds_.num_of_classes();
+  if (verbose_) {
+    std::cout << "dataset train/test/val size is:" << ds_.train_data().size() << "/" << ds_.test_data().size() << "/"
+              << ds_.val_data().size() << std::endl;
+  }
+
+  if (ds_.test_data().size() == 0) {
+    std::cout << "No relevant data was found in " << data_dir_ << std::endl;
+    assert(ds_.test_data().size() != 0);
+  }
+
+  return ret;
+}
+
+float NetRunner::GetLoss() const {
+  auto outputsv = SearchOutputsForSize(1);  // Search for Loss which is a single value tensor
+  assert(outputsv != nullptr);
+  auto loss = reinterpret_cast<float *>(outputsv->MutableData());
+  return loss[0];
+}
+
+int NetRunner::TrainLoop() {
+  session_->Train();
+  float min_loss = 1000.;
+  float max_acc = 0.;
+  for (int i = 0; i < cycles_; i++) {
+    FillInputData(ds_.train_data());
+    session_->RunGraph(nullptr, verbose_ ? after_callback : nullptr);
+    float loss = GetLoss();
+    if (min_loss > loss) min_loss = loss;
+
+    if (save_checkpoint_ != 0 && (i + 1) % save_checkpoint_ == 0) {
+      auto cpkt_fn = ms_file_.substr(0, ms_file_.find_last_of('.')) + "_trained_" + std::to_string(i + 1) + ".ms";
+      session_->SaveToFile(cpkt_fn);
+    }
+
+    std::cout << i + 1 << ": Loss is " << loss << " [min=" << min_loss << "]" << std::endl;
+    if ((i + 1) % 20 == 0) {
+      float acc = CalculateAccuracy(ds_.test_data());
+      if (max_acc < acc) max_acc = acc;
+      std::cout << "accuracy on test data = " << acc << " max accuracy = " << max_acc << std::endl;
+      if (acc > 0.9) return 0;
+    }
+  }
+  return 0;
+}
+
+int NetRunner::Main() {
+  InitAndFigureInputs();
+
+  InitDB();
+
+  TrainLoop();
+
+  float acc = CalculateAccuracy(ds_.val_data());
+  std::cout << "accuracy on validation data = " << acc << std::endl;
+
+  if (cycles_ > 0) {
+    auto trained_fn = ms_file_.substr(0, ms_file_.find_last_of('.')) + "_trained.ms";
+    session_->SaveToFile(trained_fn);
+  }
+  return 0;
+}
+
+void NetRunner::Usage() {
+  std::cout << "Usage: net_runner -f <.ms model file> -d <data_dir> [-c <num of training cycles>] "
+            << "[-v (verbose mode)] [-s <save checkpoint every X iterations>]" << std::endl;
+}
+
+bool NetRunner::ReadArgs(int argc, char *argv[]) {
+  int opt;
+  while ((opt = getopt(argc, argv, "f:e:d:s:ihc:v")) != -1) {
+    switch (opt) {
+      case 'f':
+        ms_file_ = std::string(optarg);
+        break;
+      case 'e':
+        cycles_ = atoi(optarg);
+        break;
+      case 'd':
+        data_dir_ = std::string(optarg);
+        break;
+      case 'v':
+        verbose_ = true;
+        break;
+      case 's':
+        save_checkpoint_ = atoi(optarg);
+        break;
+      case 'h':
+      default:
+        Usage();
+        return false;
+    }
+  }
+  return true;
+}
+
+int main(int argc, char **argv) {
+  NetRunner nr;
+
+  if (nr.ReadArgs(argc, argv)) {
+    nr.Main();
+  } else {
+    return -1;
+  }
+  return 0;
+}

mindspore/lite/examples/transfer_learning/src/net_runner.h

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_EXAMPLES_TRANSFER_LEARNING_SRC_NET_RUNNER_H_
+#define MINDSPORE_LITE_EXAMPLES_TRANSFER_LEARNING_SRC_NET_RUNNER_H_
+
+#include <tuple>
+#include <filesystem>
+#include <map>
+#include <vector>
+#include <string>
+#include "include/train_session.h"
+#include "include/ms_tensor.h"
+#include "src/dataset.h"
+
+class NetRunner {
+ public:
+  int Main();
+  bool ReadArgs(int argc, char *argv[]);
+  ~NetRunner();
+
+ private:
+  void Usage();
+  void InitAndFigureInputs();
+  int InitDB();
+  int TrainLoop();
+  std::vector<int> FillInputData(const std::vector<DataLabelTuple> &dataset, int serially = -1) const;
+  float CalculateAccuracy(const std::vector<DataLabelTuple> &dataset) const;
+  float GetLoss() const;
+  mindspore::tensor::MSTensor *SearchOutputsForSize(size_t size) const;
+
+  DataSet ds_;
+  mindspore::session::TrainSession *session_ = nullptr;
+
+  std::string ms_file_ = "";
+  std::string data_dir_ = "";
+  size_t data_size_ = 0;
+  size_t batch_size_ = 0;
+  unsigned int cycles_ = 100;
+  bool verbose_ = false;
+  int data_index_ = 0;
+  int label_index_ = -1;
+  int num_of_classes_ = 0;
+  int save_checkpoint_ = 0;
+};
+
+#endif  // MINDSPORE_LITE_EXAMPLES_TRANSFER_LEARNING_SRC_NET_RUNNER_H_

mindspore/lite/nnacl/fp32_grad/arithmetic_grad.c

@@ -34,12 +34,12 @@ void MaximumByAxes(const float *input0, const float *input1, const float *dy, co
                    const int *input1_dims, const int *dy_dims, float *output0, float *output1, int num_dims) {
   int num_output0 = 1;
   int num_output1 = 1;
-  int same_shape = 1;
+  bool same_shape = true;
   for (int idx = 0; idx < num_dims; ++idx) {
     num_output0 *= input0_dims[idx];
     num_output1 *= input1_dims[idx];
     if (input0_dims[idx] != input1_dims[idx]) {
-      same_shape = 0;
+      same_shape = false;
     }
   }
 
@@ -84,12 +84,12 @@ void MinimumByAxes(const float *input0, const float *input1, const float *dy, co
                    const int *input1_dims, const int *dy_dims, float *output0, float *output1, int num_dims) {
   int num_output0 = 1;
   int num_output1 = 1;
-  int same_shape = 1;
+  bool same_shape = true;
   for (int idx = 0; idx < num_dims; ++idx) {
     num_output0 *= input0_dims[idx];
     num_output1 *= input1_dims[idx];
     if (input0_dims[idx] != input1_dims[idx]) {
-      same_shape = 0;
+      same_shape = false;
     }
   }
 

mindspore/lite/nnacl/fp32_grad/smooth_l1_loss.h

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_NNACL_FP32_SMOOTH_L1_LOSS_PARAMETER_H_
+#define MINDSPORE_LITE_NNACL_FP32_SMOOTH_L1_LOSS_PARAMETER_H_
+
+#include "nnacl/op_base.h"
+
+typedef struct SmoothL1LossParameter {
+  OpParameter op_parameter_;
+  float beta_;
+} SmoothL1LossParameter;
+
+#endif  // MINDSPORE_LITE_NNACL_FP32_SMOOTH_L1_LOSS_PARAMETER_H_

mindspore/lite/schema/model.fbs

@@ -254,7 +254,11 @@ union PrimitiveType {
     Assert,
     Adder,
     SparseSoftmaxCrossEntropy,
-    Reciprocal,
+    SmoothL1Loss,
+    SmoothL1LossGrad,
+    SigmoidCrossEntropyWithLogits,
+    SigmoidCrossEntropyWithLogitsGrad,
+    Reciprocal
 }
 
 enum QuantType: int {

mindspore/lite/schema/ops.fbs

@@ -1204,5 +1204,21 @@ table Assert {
     summarize : int;
 }
 
+table SmoothL1Loss {
+    beta : float;
+}
+
+table SmoothL1LossGrad {
+    beta : float;
+}
+
+table SigmoidCrossEntropyWithLogits {
+    beta : float;
+}
+
+table SigmoidCrossEntropyWithLogitsGrad {
+    beta : float;
+}
+
 table Reciprocal {
 }

mindspore/lite/src/ops/addn.cc

@@ -88,11 +88,23 @@ int AddN::InferShape(std::vector<Tensor *> inputs, std::vector<Tensor *> outputs
   if (!infer_flag()) {
     return RET_INFER_INVALID;
   }
-  output->set_shape(input->shape());
+
+  size_t max_dims = inputs.at(0)->shape().size();
+  size_t max_dims_idx = 0;
+
+  // determine max_dims
+  for (size_t i = 1; i < inputs.size(); ++i) {
+    if (inputs.at(i)->shape().size() > max_dims) {
+      max_dims = inputs.at(i)->shape().size();
+      max_dims_idx = 0;
+    }
+  }
+  output->set_shape(inputs.at(max_dims_idx)->shape());
 
   // make sure all elements have the same size or 1 (broadcasting) in all dimensions
   for (size_t i = 1; i < inputs.size(); ++i) {
-    if (inputs.at(i)->shape().size() != inputs.at(0)->shape().size()) {
+    if ((inputs.at(i)->shape().size() != max_dims) &&
+        (inputs.at(i)->ElementsNum() != inputs.at(max_dims_idx)->ElementsNum())) {
       MS_LOG(ERROR) << "AddN inputs shape is not equal!";
       return RET_INPUT_TENSOR_ERROR;
     }
@@ -103,18 +115,24 @@ int AddN::InferShape(std::vector<Tensor *> inputs, std::vector<Tensor *> outputs
   }
 
   for (size_t d = 0; d < input->shape().size(); ++d) {
-    int max_dim = input->shape().at(d);
-    for (size_t i = 1; i < inputs.size(); ++i) {
-      if (inputs.at(i)->shape().at(d) > max_dim) {
-        max_dim = inputs.at(i)->shape().at(d);
+    size_t max_dim = 0;
+    for (size_t i = 0; i < inputs.size(); ++i) {
+      size_t shift = max_dims - inputs.at(i)->shape().size();
+      size_t dim = (i < shift) ? 1 : inputs.at(i)->shape().at(d);
+      if (dim > max_dim) {
+        max_dim = dim;
       }
     }
-    for (size_t i = 1; i < inputs.size(); ++i) {
-      if ((inputs.at(0)->shape().at(d) != max_dim) && (inputs.at(0)->shape().at(d) != 1)) {
+#ifndef SUPPORT_TRAIN
+    for (size_t i = 0; i < inputs.size(); ++i) {
+      size_t shift = max_dims - inputs.at(i)->shape().size();
+      size_t dim = (i < shift) ? 1 : inputs.at(i)->shape().at(d);
+      if ((dim != max_dim) && (dim != 1)) {
         MS_LOG(ERROR) << "AddN inputs shape is not equal!";
         return RET_INPUT_TENSOR_ERROR;
       }
     }
+#endif
     output->shape()[d] = max_dim;  // set the biggest dimension in the output tensor
   }
 

mindspore/lite/src/ops/maximum_grad.cc

@@ -118,6 +118,8 @@ int MaximumGrad::InferShape(std::vector<Tensor *> inputs_, std::vector<Tensor *>
   dx2->set_shape(x2->shape());
   dx1->set_data_type(dy->data_type());
   dx2->set_data_type(dy->data_type());
+  dx1->set_format(dy->format());
+  dx2->set_format(dy->format());
   return RET_OK;
 }
 }  // namespace lite

mindspore/lite/src/ops/minimum_grad.cc

@@ -72,5 +72,57 @@ int MinimumGrad::UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuf
   return RET_OK;
 }
 #endif
+
+int MinimumGrad::InferShape(std::vector<Tensor *> inputs_, std::vector<Tensor *> outputs_) {
+  if (inputs_.size() != 3) {
+    MS_LOG(ERROR) << "The number of input must be 3";
+    return RET_ERROR;
+  }
+  if (outputs_.size() != 2) {
+    MS_LOG(ERROR) << "The number of output must be 2";
+    return RET_ERROR;
+  }
+
+  auto x1 = inputs_[0];
+  auto x2 = inputs_[1];
+  auto dy = inputs_[2];
+  auto dx1 = outputs_[0];
+  auto dx2 = outputs_[1];
+
+  MS_ASSERT(dy != nullptr);
+  MS_ASSERT(x1 != nullptr);
+  MS_ASSERT(x2 != nullptr);
+  MS_ASSERT(dx1 != nullptr);
+  MS_ASSERT(dx2 != nullptr);
+  if (!infer_flag()) {
+    return RET_OK;
+  }
+
+  auto inShape0 = x1->shape();
+  auto inShape1 = x2->shape();
+  auto outShape = dy->shape();
+
+  ndim_ = outShape.size();
+  x1_shape_.resize(ndim_);
+  x2_shape_.resize(ndim_);
+  dy_shape_.resize(ndim_);
+  auto fillDimNum0 = outShape.size() - inShape0.size();
+  auto fillDimNum1 = outShape.size() - inShape1.size();
+  int j0 = 0;
+  int j1 = 0;
+  for (unsigned int i = 0; i < outShape.size(); i++) {
+    x1_shape_[i] = (i < fillDimNum0) ? 1 : inShape0[j0++];
+    x2_shape_[i] = (i < fillDimNum1) ? 1 : inShape1[j1++];
+    dy_shape_[i] = outShape[i];
+  }
+
+  dx1->set_shape(x1->shape());
+  dx2->set_shape(x2->shape());
+  dx1->set_data_type(dy->data_type());
+  dx2->set_data_type(dy->data_type());
+  dx1->set_format(dy->format());
+  dx2->set_format(dy->format());
+  return RET_OK;
+}
 }  // namespace lite
 }  // namespace mindspore

mindspore/lite/src/ops/minimum_grad.h

@@ -35,9 +35,9 @@ class MinimumGrad : public ArithmeticGrad {
   int UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) override;
 #else
   MinimumGrad() = default;
-
   int UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) override;
 #endif
+  int InferShape(std::vector<lite::Tensor *> inputs_, std::vector<lite::Tensor *> outputs_) override;
 };
 }  // namespace lite
 }  // namespace mindspore

mindspore/lite/src/ops/primitive_c.cc

@@ -177,6 +177,10 @@
 #include "src/ops/assign_add.h"
 #include "src/ops/binary_cross_entropy.h"
 #include "src/ops/binary_cross_entropy_grad.h"
+#include "src/ops/smooth_l1_loss.h"
+#include "src/ops/smooth_l1_loss_grad.h"
+#include "src/ops/sigmoid_cross_entropy_with_logits.h"
+#include "src/ops/sigmoid_cross_entropy_with_logits_grad.h"
 #endif
 #endif
 namespace mindspore {
@@ -626,6 +630,14 @@ std::shared_ptr<PrimitiveC> PrimitiveC::Create(const Primitive &prim, const std:
     return NewPrimitiveC<BinaryCrossEntropy>(prim, inputs, quantType);
   } else if (op_type == "BinaryCrossEntropyGrad") {
     return NewPrimitiveC<BinaryCrossEntropyGrad>(prim, inputs, quantType);
+  } else if (op_type == "SmoothL1Loss") {
+    return NewPrimitiveC<SmoothL1Loss>(prim, inputs, quantType);
+  } else if (op_type == "SmoothL1LossGrad") {
+    return NewPrimitiveC<SmoothL1LossGrad>(prim, inputs, quantType);
+  } else if (op_type == "SigmoidCrossEntropyWithLogits") {
+    return NewPrimitiveC<SigmoidCrossEntropyWithLogits>(prim, inputs, quantType);
+  } else if (op_type == "SigmoidCrossEntropyWithLogitsGrad") {
+    return NewPrimitiveC<SigmoidCrossEntropyWithLogitsGrad>(prim, inputs, quantType);
 #else
   } else if (op_type == "Conv2DBackpropInput") {
     return NewPrimitiveC<DeConv2D>(prim, inputs, quantType);
@@ -955,6 +967,14 @@ PrimitiveC *PrimitiveC::Create(mindspore::schema::PrimitiveT *primitive) {
       return new (std::nothrow) MaximumGrad(primitive);
     case schema::PrimitiveType_MinimumGrad:
       return new (std::nothrow) MinimumGrad(primitive);
+    case schema::PrimitiveType_SmoothL1Loss:
+      return new (std::nothrow) SmoothL1Loss(primitive);
+    case schema::PrimitiveType_SmoothL1LossGrad:
+      return new (std::nothrow) SmoothL1LossGrad(primitive);
+    case schema::PrimitiveType_SigmoidCrossEntropyWithLogits:
+      return new (std::nothrow) SigmoidCrossEntropyWithLogits(primitive);
+    case schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad:
+      return new (std::nothrow) SigmoidCrossEntropyWithLogitsGrad(primitive);
 #endif
     default:
       MS_LOG(ERROR) << "Unsupported primitive type in Create : " << schema::EnumNamePrimitiveType(op_type);

mindspore/lite/src/ops/sigmoid_cross_entropy_with_logits.cc

@@ -0,0 +1,100 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "src/ops/sigmoid_cross_entropy_with_logits.h"
+#ifndef PRIMITIVE_WRITEABLE
+#include "src/ops/ops_register.h"
+#endif
+
+namespace mindspore {
+namespace lite {
+#ifdef PRIMITIVE_WRITEABLE
+int SigmoidCrossEntropyWithLogits::UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) {
+  if (this->primitive_ == nullptr) {
+    this->primitive_ = new (std::nothrow) schema::PrimitiveT;
+    if (this->primitive_ == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT failed";
+      return RET_ERROR;
+    }
+    this->primitive_->value.type = schema::PrimitiveType_SigmoidCrossEntropyWithLogits;
+  }
+  if (this->primitive_->value.type != schema::PrimitiveType_SigmoidCrossEntropyWithLogits) {
+    MS_LOG(ERROR) << "Primitive type is error :" << this->primitive_->value.type;
+    return RET_ERROR;
+  }
+  if (this->primitive_->value.value == nullptr) {
+    auto attr = std::make_unique<schema::SigmoidCrossEntropyWithLogitsT>();
+    if (attr == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+
+    this->primitive_->value.value = attr.release();
+    if (this->primitive_->value.value == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+  }
+  return RET_OK;
+}
+#else
+int SigmoidCrossEntropyWithLogits::UnPackToFlatBuilder(const schema::Primitive *primitive,
+                                                       flatbuffers::FlatBufferBuilder *fbb) {
+  MS_ASSERT(nullptr != primitive);
+  MS_ASSERT(nullptr != fbb);
+  auto attr = primitive->value_as_SigmoidCrossEntropyWithLogits();
+  if (attr == nullptr) {
+    MS_LOG(ERROR) << "value_as_SigmoidCrossEntropyWithLogits return nullptr";
+    return RET_ERROR;
+  }
+  auto val_offset = schema::CreateSigmoidCrossEntropyWithLogits(*fbb);
+  auto prim_offset = schema::CreatePrimitive(*fbb, schema::PrimitiveType_SigmoidCrossEntropyWithLogits, val_offset.o);
+  fbb->Finish(prim_offset);
+  return RET_OK;
+}
+
+PrimitiveC *SigmoidCrossEntropyWithLogitsCreator(const schema::Primitive *primitive) {
+  return PrimitiveC::NewPrimitiveC<SigmoidCrossEntropyWithLogits>(primitive);
+}
+Registry SigmoidCrossEntropyWithLogitsRegistry(schema::PrimitiveType_SigmoidCrossEntropyWithLogits,
+                                               SigmoidCrossEntropyWithLogitsCreator);
+#endif
+
+int SigmoidCrossEntropyWithLogits::InferShape(std::vector<lite::Tensor *> inputs, std::vector<lite::Tensor *> outputs) {
+  if (inputs.size() != 2) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogits should have 2 input tensors";
+    return RET_ERROR;
+  }
+
+  if (outputs.size() != 1) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogits should have 1 output tensors";
+    return RET_ERROR;
+  }
+
+  if (inputs[0]->ElementsNum() != inputs[1]->ElementsNum()) {
+    MS_LOG(ERROR) << "error input data size!";
+    return RET_ERROR;
+  }
+
+  auto *out = outputs.front();
+  MS_ASSERT(out != nullptr);
+  out->set_data_type(inputs[0]->data_type());
+  out->set_format(inputs[0]->format());
+  out->set_shape(inputs[0]->shape());
+
+  return RET_OK;
+}
+}  // namespace lite
+}  // namespace mindspore

mindspore/lite/src/ops/sigmoid_cross_entropy_with_logits.h

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_OPS_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_H_
+#define MINDSPORE_LITE_SRC_OPS_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_H_
+
+#include <vector>
+#include <set>
+#include <cmath>
+#include <memory>
+
+#include "src/ops/primitive_c.h"
+
+namespace mindspore {
+namespace lite {
+class SigmoidCrossEntropyWithLogits : public PrimitiveC {
+ public:
+  SigmoidCrossEntropyWithLogits() = default;
+  ~SigmoidCrossEntropyWithLogits() = default;
+#ifdef PRIMITIVE_WRITEABLE
+  MS_DECLARE_PARENT(SigmoidCrossEntropyWithLogits, PrimitiveC);
+  explicit SigmoidCrossEntropyWithLogits(schema::PrimitiveT *primitive) : PrimitiveC(primitive) {}
+  int UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) override;
+#else
+  int UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) override;
+#endif
+  int InferShape(std::vector<lite::Tensor *> inputs_, std::vector<lite::Tensor *> outputs_) override;
+};
+}  // namespace lite
+}  // namespace mindspore
+
+#endif  // MINDSPORE_LITE_SRC_OPS_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_H_

mindspore/lite/src/ops/sigmoid_cross_entropy_with_logits_grad.cc

@@ -0,0 +1,102 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "src/ops/sigmoid_cross_entropy_with_logits_grad.h"
+#ifndef PRIMITIVE_WRITEABLE
+#include "src/ops/ops_register.h"
+#endif
+
+namespace mindspore {
+namespace lite {
+#ifdef PRIMITIVE_WRITEABLE
+int SigmoidCrossEntropyWithLogitsGrad::UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) {
+  if (this->primitive_ == nullptr) {
+    this->primitive_ = new (std::nothrow) schema::PrimitiveT;
+    if (this->primitive_ == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT failed";
+      return RET_ERROR;
+    }
+    this->primitive_->value.type = schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad;
+  }
+  if (this->primitive_->value.type != schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad) {
+    MS_LOG(ERROR) << "Primitive type is error :" << this->primitive_->value.type;
+    return RET_ERROR;
+  }
+  if (this->primitive_->value.value == nullptr) {
+    auto attr = std::make_unique<schema::SigmoidCrossEntropyWithLogitsGradT>();
+    if (attr == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+
+    this->primitive_->value.value = attr.release();
+    if (this->primitive_->value.value == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+  }
+  return RET_OK;
+}
+#else
+int SigmoidCrossEntropyWithLogitsGrad::UnPackToFlatBuilder(const schema::Primitive *primitive,
+                                                           flatbuffers::FlatBufferBuilder *fbb) {
+  MS_ASSERT(nullptr != primitive);
+  MS_ASSERT(nullptr != fbb);
+  auto attr = primitive->value_as_SigmoidCrossEntropyWithLogitsGrad();
+  if (attr == nullptr) {
+    MS_LOG(ERROR) << "value_as_SigmoidCrossEntropyWithLogitsGrad return nullptr";
+    return RET_ERROR;
+  }
+  auto val_offset = schema::CreateSigmoidCrossEntropyWithLogitsGrad(*fbb);
+  auto prim_offset =
+    schema::CreatePrimitive(*fbb, schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad, val_offset.o);
+  fbb->Finish(prim_offset);
+  return RET_OK;
+}
+
+PrimitiveC *SigmoidCrossEntropyWithLogitsGradCreator(const schema::Primitive *primitive) {
+  return PrimitiveC::NewPrimitiveC<SigmoidCrossEntropyWithLogitsGrad>(primitive);
+}
+Registry SigmoidCrossEntropyWithLogitsGradRegistry(schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad,
+                                                   SigmoidCrossEntropyWithLogitsGradCreator);
+#endif
+
+int SigmoidCrossEntropyWithLogitsGrad::InferShape(std::vector<lite::Tensor *> inputs,
+                                                  std::vector<lite::Tensor *> outputs) {
+  if (inputs.size() != 3) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogitsGrad should have 3 input tensors";
+    return RET_ERROR;
+  }
+
+  if (outputs.size() != 1) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogitsGrad should have 1 output tensors";
+    return RET_ERROR;
+  }
+
+  if (inputs[0]->ElementsNum() != inputs[1]->ElementsNum() || inputs[0]->ElementsNum() != inputs[2]->ElementsNum()) {
+    MS_LOG(ERROR) << "error input data size!";
+    return RET_ERROR;
+  }
+
+  auto *out = outputs.front();
+  MS_ASSERT(out != nullptr);
+  out->set_data_type(inputs[0]->data_type());
+  out->set_format(inputs[0]->format());
+  out->set_shape(inputs[0]->shape());
+
+  return RET_OK;
+}
+}  // namespace lite
+}  // namespace mindspore

mindspore/lite/src/ops/sigmoid_cross_entropy_with_logits_grad.h

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_OPS_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_GRAD_H_
+#define MINDSPORE_LITE_SRC_OPS_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_GRAD_H_
+
+#include <vector>
+#include <set>
+#include <cmath>
+#include <memory>
+
+#include "src/ops/primitive_c.h"
+
+namespace mindspore {
+namespace lite {
+class SigmoidCrossEntropyWithLogitsGrad : public PrimitiveC {
+ public:
+  SigmoidCrossEntropyWithLogitsGrad() = default;
+  ~SigmoidCrossEntropyWithLogitsGrad() = default;
+#ifdef PRIMITIVE_WRITEABLE
+  MS_DECLARE_PARENT(SigmoidCrossEntropyWithLogitsGrad, PrimitiveC);
+  explicit SigmoidCrossEntropyWithLogitsGrad(schema::PrimitiveT *primitive) : PrimitiveC(primitive) {}
+  int UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) override;
+#else
+  int UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) override;
+#endif
+  int InferShape(std::vector<lite::Tensor *> inputs_, std::vector<lite::Tensor *> outputs_) override;
+};
+}  // namespace lite
+}  // namespace mindspore
+
+#endif  // MINDSPORE_LITE_SRC_OPS_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_GRAD_H_

mindspore/lite/src/ops/smooth_l1_loss.cc

@@ -0,0 +1,101 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "src/ops/smooth_l1_loss.h"
+#ifndef PRIMITIVE_WRITEABLE
+#include "src/ops/ops_register.h"
+#endif
+
+namespace mindspore {
+namespace lite {
+#ifdef PRIMITIVE_WRITEABLE
+float SmoothL1Loss::GetBeta() const { return this->primitive_->value.AsSmoothL1Loss()->beta; }
+int SmoothL1Loss::UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) {
+  if (this->primitive_ == nullptr) {
+    this->primitive_ = new (std::nothrow) schema::PrimitiveT;
+    if (this->primitive_ == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT failed";
+      return RET_ERROR;
+    }
+    this->primitive_->value.type = schema::PrimitiveType_SmoothL1Loss;
+  }
+  if (this->primitive_->value.type != schema::PrimitiveType_SmoothL1Loss) {
+    MS_LOG(ERROR) << "Primitive type is error :" << this->primitive_->value.type;
+    return RET_ERROR;
+  }
+  if (this->primitive_->value.value == nullptr) {
+    auto attr = std::make_unique<schema::SmoothL1LossT>();
+    if (attr == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+    attr->beta = GetValue<float>(prim.GetAttr("beta"));
+
+    this->primitive_->value.value = attr.release();
+    if (this->primitive_->value.value == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+  }
+  return RET_OK;
+}
+#else
+float SmoothL1Loss::GetBeta() const { return this->primitive_->value_as_SmoothL1Loss()->beta(); }
+int SmoothL1Loss::UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) {
+  MS_ASSERT(nullptr != primitive);
+  MS_ASSERT(nullptr != fbb);
+  auto attr = primitive->value_as_SmoothL1Loss();
+  if (attr == nullptr) {
+    MS_LOG(ERROR) << "value_as_SmoothL1Loss return nullptr";
+    return RET_ERROR;
+  }
+  auto val_offset = schema::CreateSmoothL1Loss(*fbb, attr->beta());
+  auto prim_offset = schema::CreatePrimitive(*fbb, schema::PrimitiveType_SmoothL1Loss, val_offset.o);
+  fbb->Finish(prim_offset);
+  return RET_OK;
+}
+
+PrimitiveC *SmoothL1LossCreator(const schema::Primitive *primitive) {
+  return PrimitiveC::NewPrimitiveC<SmoothL1Loss>(primitive);
+}
+Registry SmoothL1LossRegistry(schema::PrimitiveType_SmoothL1Loss, SmoothL1LossCreator);
+#endif
+
+int SmoothL1Loss::InferShape(std::vector<lite::Tensor *> inputs, std::vector<lite::Tensor *> outputs) {
+  if (inputs.size() != 2) {
+    MS_LOG(ERROR) << "SmoothL1Loss should have 2 input tensors";
+    return RET_ERROR;
+  }
+
+  if (outputs.size() != 1) {
+    MS_LOG(ERROR) << "SmoothL1Loss should have 1 output tensors";
+    return RET_ERROR;
+  }
+
+  if (inputs[0]->ElementsNum() != inputs[1]->ElementsNum()) {
+    MS_LOG(ERROR) << "error input data size!";
+    return RET_ERROR;
+  }
+
+  auto *out = outputs.front();
+  MS_ASSERT(out != nullptr);
+  out->set_data_type(inputs[0]->data_type());
+  out->set_format(inputs[0]->format());
+  out->set_shape(inputs[0]->shape());
+
+  return RET_OK;
+}
+}  // namespace lite
+}  // namespace mindspore

mindspore/lite/src/ops/smooth_l1_loss.h

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_OPS_SMOOTH_L1_LOSS_H_
+#define MINDSPORE_LITE_SRC_OPS_SMOOTH_L1_LOSS_H_
+
+#include <vector>
+#include <set>
+#include <cmath>
+#include <memory>
+
+#include "src/ops/primitive_c.h"
+
+namespace mindspore {
+namespace lite {
+class SmoothL1Loss : public PrimitiveC {
+ public:
+  SmoothL1Loss() = default;
+  ~SmoothL1Loss() = default;
+#ifdef PRIMITIVE_WRITEABLE
+  MS_DECLARE_PARENT(SmoothL1Loss, PrimitiveC);
+  explicit SmoothL1Loss(schema::PrimitiveT *primitive) : PrimitiveC(primitive) {}
+  int UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) override;
+#else
+  int UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) override;
+#endif
+  int InferShape(std::vector<lite::Tensor *> inputs_, std::vector<lite::Tensor *> outputs_) override;
+  float GetBeta() const;
+};
+}  // namespace lite
+}  // namespace mindspore
+
+#endif  // MINDSPORE_LITE_SRC_OPS_SMOOTH_L1_LOSS_H_

mindspore/lite/src/ops/smooth_l1_loss_grad.cc

@@ -0,0 +1,101 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "src/ops/smooth_l1_loss_grad.h"
+#ifndef PRIMITIVE_WRITEABLE
+#include "src/ops/ops_register.h"
+#endif
+
+namespace mindspore {
+namespace lite {
+#ifdef PRIMITIVE_WRITEABLE
+float SmoothL1LossGrad::GetBeta() const { return this->primitive_->value.AsSmoothL1LossGrad()->beta; }
+int SmoothL1LossGrad::UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) {
+  if (this->primitive_ == nullptr) {
+    this->primitive_ = new (std::nothrow) schema::PrimitiveT;
+    if (this->primitive_ == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT failed";
+      return RET_ERROR;
+    }
+    this->primitive_->value.type = schema::PrimitiveType_SmoothL1LossGrad;
+  }
+  if (this->primitive_->value.type != schema::PrimitiveType_SmoothL1LossGrad) {
+    MS_LOG(ERROR) << "Primitive type is error :" << this->primitive_->value.type;
+    return RET_ERROR;
+  }
+  if (this->primitive_->value.value == nullptr) {
+    auto attr = std::make_unique<schema::SmoothL1LossGradT>();
+    if (attr == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+    attr->beta = GetValue<float>(prim.GetAttr("beta"));
+
+    this->primitive_->value.value = attr.release();
+    if (this->primitive_->value.value == nullptr) {
+      MS_LOG(ERROR) << "new primitiveT value failed";
+      return RET_ERROR;
+    }
+  }
+  return RET_OK;
+}
+#else
+float SmoothL1LossGrad::GetBeta() const { return this->primitive_->value_as_SmoothL1LossGrad()->beta(); }
+int SmoothL1LossGrad::UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) {
+  MS_ASSERT(nullptr != primitive);
+  MS_ASSERT(nullptr != fbb);
+  auto attr = primitive->value_as_SmoothL1LossGrad();
+  if (attr == nullptr) {
+    MS_LOG(ERROR) << "value_as_SmoothL1LossGrad return nullptr";
+    return RET_ERROR;
+  }
+  auto val_offset = schema::CreateSmoothL1LossGrad(*fbb, attr->beta());
+  auto prim_offset = schema::CreatePrimitive(*fbb, schema::PrimitiveType_SmoothL1LossGrad, val_offset.o);
+  fbb->Finish(prim_offset);
+  return RET_OK;
+}
+
+PrimitiveC *SmoothL1LossGradCreator(const schema::Primitive *primitive) {
+  return PrimitiveC::NewPrimitiveC<SmoothL1LossGrad>(primitive);
+}
+Registry SmoothL1LossGradRegistry(schema::PrimitiveType_SmoothL1LossGrad, SmoothL1LossGradCreator);
+#endif
+
+int SmoothL1LossGrad::InferShape(std::vector<lite::Tensor *> inputs, std::vector<lite::Tensor *> outputs) {
+  if (inputs.size() != 3) {
+    MS_LOG(ERROR) << "SmoothL1LossGrad should have 3 input tensors";
+    return RET_ERROR;
+  }
+
+  if (outputs.size() != 1) {
+    MS_LOG(ERROR) << "SmoothL1LossGrad should have 1 output tensors";
+    return RET_ERROR;
+  }
+
+  if (inputs[0]->ElementsNum() != inputs[1]->ElementsNum() || inputs[0]->ElementsNum() != inputs[2]->ElementsNum()) {
+    MS_LOG(ERROR) << "error input data size!";
+    return RET_ERROR;
+  }
+
+  auto *out = outputs.front();
+  MS_ASSERT(out != nullptr);
+  out->set_data_type(inputs[0]->data_type());
+  out->set_format(inputs[0]->format());
+  out->set_shape(inputs[0]->shape());
+
+  return RET_OK;
+}
+}  // namespace lite
+}  // namespace mindspore

mindspore/lite/src/ops/smooth_l1_loss_grad.h

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_OPS_SMOOTH_L1_LOSS_GRAD_H_
+#define MINDSPORE_LITE_SRC_OPS_SMOOTH_L1_LOSS_GRAD_H_
+
+#include <vector>
+#include <set>
+#include <cmath>
+#include <memory>
+
+#include "src/ops/primitive_c.h"
+
+namespace mindspore {
+namespace lite {
+class SmoothL1LossGrad : public PrimitiveC {
+ public:
+  SmoothL1LossGrad() = default;
+  ~SmoothL1LossGrad() = default;
+#ifdef PRIMITIVE_WRITEABLE
+  MS_DECLARE_PARENT(SmoothL1LossGrad, PrimitiveC);
+  explicit SmoothL1LossGrad(schema::PrimitiveT *primitive) : PrimitiveC(primitive) {}
+  int UnPackAttr(const Primitive &prim, const std::vector<AnfNodePtr> &inputs) override;
+#else
+  int UnPackToFlatBuilder(const schema::Primitive *primitive, flatbuffers::FlatBufferBuilder *fbb) override;
+#endif
+  int InferShape(std::vector<lite::Tensor *> inputs_, std::vector<lite::Tensor *> outputs_) override;
+  float GetBeta() const;
+};
+}  // namespace lite
+}  // namespace mindspore
+
+#endif  // MINDSPORE_LITE_SRC_OPS_SMOOTH_L1_LOSS_GRAD_H_

mindspore/lite/src/runtime/kernel/arm/fp32_grad/arithmetic_grad.cc

@@ -183,7 +183,6 @@ void ArithmeticGradCPUKernel::ArithmeticGradDiv2L(float *dy, int dy_size, float
 
 void ArithmeticGradCPUKernel::ArithmeticGradMaximum(float *dy, int dy_size, float *dx1, int dx1_size, float *dx2,
                                                     int dx2_size) {
-  // For some reason, input order is x0, x1, dy
   auto x1 = reinterpret_cast<float *>(in_tensors_[0]->MutableData());
   auto x2 = reinterpret_cast<float *>(in_tensors_[1]->MutableData());
   dy = reinterpret_cast<float *>(in_tensors_[2]->MutableData());
@@ -194,13 +193,12 @@ void ArithmeticGradCPUKernel::ArithmeticGradMaximum(float *dy, int dy_size, floa
 
 void ArithmeticGradCPUKernel::ArithmeticGradMinimum(float *dy, int dy_size, float *dx1, int dx1_size, float *dx2,
                                                     int dx2_size) {
-  // For some reason, input order is x0, x1, dy
   auto x1 = reinterpret_cast<float *>(in_tensors_[0]->MutableData());
   auto x2 = reinterpret_cast<float *>(in_tensors_[1]->MutableData());
   dy = reinterpret_cast<float *>(in_tensors_[2]->MutableData());
 
-  MinimumByAxes(x1, x2, dy, arithmeticParameter_->out_shape_, arithmeticParameter_->in_shape0_,
-                arithmeticParameter_->in_shape1_, dx1, dx2, arithmeticParameter_->ndim_);
+  MinimumByAxes(x1, x2, dy, arithmeticParameter_->in_shape0_, arithmeticParameter_->in_shape1_,
+                arithmeticParameter_->out_shape_, dx1, dx2, arithmeticParameter_->ndim_);
 }
 
 int ArithmeticGradCPUKernel::ReSize() { return RET_OK; }
@@ -212,7 +210,7 @@ int ArithmeticGradCPUKernel::Execute(int task_id) {
 
   size_t dy_size = in_tensors_.at(0)->ElementsNum();
   size_t dx1_size = out_tensors_.at(0)->ElementsNum();
-  size_t dx2_size = out_tensors_[1]->ElementsNum();
+  size_t dx2_size = out_tensors_.at(1)->ElementsNum();
   (this->*arithmetic_grad_)(dy, dy_size, dx1, dx1_size, dx2, dx2_size);
   return RET_OK;
 }

mindspore/lite/src/runtime/kernel/arm/fp32_grad/sigmoid_cross_entropy_with_logits.cc

@@ -0,0 +1,97 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/runtime/kernel/arm/fp32_grad/sigmoid_cross_entropy_with_logits.h"
+#include <math.h>
+#include "src/kernel_registry.h"
+#include "include/errorcode.h"
+#include "src/runtime/runtime_api.h"
+
+using mindspore::lite::KernelRegistrar;
+using mindspore::lite::RET_ERROR;
+using mindspore::lite::RET_OK;
+using mindspore::schema::PrimitiveType_SigmoidCrossEntropyWithLogits;
+
+namespace mindspore::kernel {
+
+int SigmoidCrossEntropyWithLogitsCPUKernel::ReSize() { return RET_OK; }
+
+int SigmoidCrossEntropyWithLogitsCPUKernel::Execute(int task_id) {
+  auto logits = reinterpret_cast<float *>(in_tensors_.at(0)->MutableData());
+  auto labels = reinterpret_cast<float *>(in_tensors_.at(1)->MutableData());
+  auto *out = reinterpret_cast<float *>(out_tensors_.at(0)->MutableData());
+  const size_t tensor_len = in_tensors_.at(0)->ElementsNum();
+
+  float zero = 0.0f;
+  float one = 1.0f;
+  float two = 2.0f;
+
+  for (uint64_t i = 0; i < tensor_len; ++i) {
+    if (logits[i] >= zero) {
+      out[i] = log1pf(exp(logits[i] - two * logits[i])) - logits[i] * (labels[i] - one);
+    } else {
+      out[i] = log1pf(exp(logits[i])) - logits[i] * labels[i];
+    }
+  }
+
+  return RET_OK;
+}
+
+int SigmoidCrossEntropyWithLogitsRun(void *cdata, int task_id) {
+  auto sig_crs_ent_kernel = reinterpret_cast<SigmoidCrossEntropyWithLogitsCPUKernel *>(cdata);
+  auto error_code = sig_crs_ent_kernel->Execute(task_id);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogits error task_id[" << task_id << "] error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SigmoidCrossEntropyWithLogitsCPUKernel::Run() {
+  int error_code = ParallelLaunch(this->context_->thread_pool_, SigmoidCrossEntropyWithLogitsRun, this, 1);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogits function error error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SigmoidCrossEntropyWithLogitsCPUKernel::Init() { return RET_OK; }
+
+kernel::LiteKernel *CpuSigmoidCrossEntropyWithLogitsFp32KernelCreator(
+  const std::vector<lite::Tensor *> &inputs, const std::vector<lite::Tensor *> &outputs, OpParameter *opParameter,
+  const lite::InnerContext *ctx, const kernel::KernelKey &desc, const mindspore::lite::PrimitiveC *primitive) {
+  MS_ASSERT(opParameter != nullptr);
+  MS_ASSERT(desc.type == schema::PrimitiveType_SigmoidCrossEntropyWithLogits);
+  auto *kernel =
+    new (std::nothrow) SigmoidCrossEntropyWithLogitsCPUKernel(opParameter, inputs, outputs, ctx, primitive);
+  if (kernel == nullptr) {
+    MS_LOG(ERROR) << "new SigmoidCrossEntropyWithLogits failed";
+    return nullptr;
+  }
+  auto ret = kernel->Init();
+  if (ret != RET_OK) {
+    MS_LOG(ERROR) << "Init kernel failed, name: " << opParameter->name_ << ", type: "
+                  << schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(opParameter->type_));
+    delete kernel;
+    return nullptr;
+  }
+  return kernel;
+}
+
+REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_SigmoidCrossEntropyWithLogits,
+           CpuSigmoidCrossEntropyWithLogitsFp32KernelCreator)
+}  // namespace mindspore::kernel

mindspore/lite/src/runtime/kernel/arm/fp32_grad/sigmoid_cross_entropy_with_logits.h

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_H_
+
+#include <vector>
+#include "src/lite_kernel.h"
+
+namespace mindspore::kernel {
+class SigmoidCrossEntropyWithLogitsCPUKernel : public LiteKernel {
+ public:
+  explicit SigmoidCrossEntropyWithLogitsCPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
+                                                  const std::vector<lite::Tensor *> &outputs,
+                                                  const lite::InnerContext *ctx,
+                                                  const mindspore::lite::PrimitiveC *primitive)
+      : LiteKernel(parameter, inputs, outputs, ctx, primitive) {}
+  ~SigmoidCrossEntropyWithLogitsCPUKernel() override {}
+  int Init() override;
+  int ReSize() override;
+  int Run() override;
+  int Execute(int task_id);
+};
+}  // namespace mindspore::kernel
+
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_H_

mindspore/lite/src/runtime/kernel/arm/fp32_grad/sigmoid_cross_entropy_with_logits_grad.cc

@@ -0,0 +1,97 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/runtime/kernel/arm/fp32_grad/sigmoid_cross_entropy_with_logits_grad.h"
+#include <math.h>
+#include "src/kernel_registry.h"
+#include "include/errorcode.h"
+#include "src/runtime/runtime_api.h"
+
+using mindspore::lite::KernelRegistrar;
+using mindspore::lite::RET_ERROR;
+using mindspore::lite::RET_OK;
+using mindspore::schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad;
+
+namespace mindspore::kernel {
+
+int SigmoidCrossEntropyWithLogitsGradCPUKernel::ReSize() { return RET_OK; }
+
+int SigmoidCrossEntropyWithLogitsGradCPUKernel::Execute(int task_id) {
+  auto logits = reinterpret_cast<float *>(in_tensors_.at(0)->MutableData());
+  auto labels = reinterpret_cast<float *>(in_tensors_.at(1)->MutableData());
+  auto dloss = reinterpret_cast<float *>(in_tensors_.at(2)->MutableData());
+  auto *out = reinterpret_cast<float *>(out_tensors_.at(0)->MutableData());
+  float zero = 0.0f;
+  float one = 1.0f;
+
+  size_t tensor_len = in_tensors_.at(0)->ElementsNum();
+  for (uint64_t i = 0; i < tensor_len; ++i) {
+    if (logits[i] >= zero) {
+      out[i] = (one / (one + expf(-logits[i])) - labels[i]) * dloss[i];
+    } else {
+      const float exp_val = expf(logits[i]);
+      out[i] = (exp_val / (one + exp_val) - labels[i]) * dloss[i];
+    }
+  }
+  return RET_OK;
+}
+
+int SigmoidCrossEntropyWithLogitsGradRun(void *cdata, int task_id) {
+  auto sig_crs_ent_kernel = reinterpret_cast<SigmoidCrossEntropyWithLogitsGradCPUKernel *>(cdata);
+  auto error_code = sig_crs_ent_kernel->Execute(task_id);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogitsGrad error task_id[" << task_id << "] error_code[" << error_code
+                  << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SigmoidCrossEntropyWithLogitsGradCPUKernel::Run() {
+  int error_code = ParallelLaunch(this->context_->thread_pool_, SigmoidCrossEntropyWithLogitsGradRun, this, 1);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SigmoidCrossEntropyWithLogitsGrad function error error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SigmoidCrossEntropyWithLogitsGradCPUKernel::Init() { return RET_OK; }
+
+kernel::LiteKernel *CpuSigmoidCrossEntropyWithLogitsGradFp32KernelCreator(
+  const std::vector<lite::Tensor *> &inputs, const std::vector<lite::Tensor *> &outputs, OpParameter *opParameter,
+  const lite::InnerContext *ctx, const kernel::KernelKey &desc, const mindspore::lite::PrimitiveC *primitive) {
+  MS_ASSERT(opParameter != nullptr);
+  MS_ASSERT(desc.type == schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad);
+  auto *kernel =
+    new (std::nothrow) SigmoidCrossEntropyWithLogitsGradCPUKernel(opParameter, inputs, outputs, ctx, primitive);
+  if (kernel == nullptr) {
+    MS_LOG(ERROR) << "new SigmoidCrossEntropyWithLogitsGradWithLogitsCPUKernel failed";
+    return nullptr;
+  }
+  auto ret = kernel->Init();
+  if (ret != RET_OK) {
+    MS_LOG(ERROR) << "Init kernel failed, name: " << opParameter->name_ << ", type: "
+                  << schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(opParameter->type_));
+    delete kernel;
+    return nullptr;
+  }
+  return kernel;
+}
+
+REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_SigmoidCrossEntropyWithLogitsGrad,
+           CpuSigmoidCrossEntropyWithLogitsGradFp32KernelCreator)
+}  // namespace mindspore::kernel

mindspore/lite/src/runtime/kernel/arm/fp32_grad/sigmoid_cross_entropy_with_logits_grad.h

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_GRAD_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_GRAD_H_
+
+#include <vector>
+#include "src/lite_kernel.h"
+
+namespace mindspore::kernel {
+class SigmoidCrossEntropyWithLogitsGradCPUKernel : public LiteKernel {
+ public:
+  explicit SigmoidCrossEntropyWithLogitsGradCPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
+                                                      const std::vector<lite::Tensor *> &outputs,
+                                                      const lite::InnerContext *ctx,
+                                                      const mindspore::lite::PrimitiveC *primitive)
+      : LiteKernel(parameter, inputs, outputs, ctx, primitive) {}
+  ~SigmoidCrossEntropyWithLogitsGradCPUKernel() override {}
+  int Init() override;
+  int ReSize() override;
+  int Run() override;
+  int Execute(int task_id);
+};
+}  // namespace mindspore::kernel
+
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SIGMOID_CROSS_ENTROPY_WITH_LOGITS_GRAD_H_

mindspore/lite/src/runtime/kernel/arm/fp32_grad/smooth_l1_loss.cc

@@ -0,0 +1,102 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/runtime/kernel/arm/fp32_grad/smooth_l1_loss.h"
+#include "src/kernel_registry.h"
+#include "include/errorcode.h"
+#include "src/runtime/runtime_api.h"
+
+using mindspore::lite::KernelRegistrar;
+using mindspore::lite::RET_ERROR;
+using mindspore::lite::RET_OK;
+using mindspore::schema::PrimitiveType_SmoothL1Loss;
+
+namespace mindspore::kernel {
+
+int SmoothL1LossCPUKernel::ReSize() { return RET_OK; }
+
+int SmoothL1LossCPUKernel::Execute(int task_id) {
+  SmoothL1LossParameter *smooth_l1_loss_param = reinterpret_cast<SmoothL1LossParameter *>(op_parameter_);
+
+  auto predict = reinterpret_cast<float *>(in_tensors_.at(0)->MutableData());
+  auto target = reinterpret_cast<float *>(in_tensors_.at(1)->MutableData());
+  auto *out = reinterpret_cast<float *>(out_tensors_.at(0)->MutableData());
+
+  const size_t tensor_len = in_tensors_.at(0)->ElementsNum();
+  const float zero = 0.0f;
+  const float half = 0.5f;
+  const float beta = smooth_l1_loss_param->beta_;
+
+  for (uint64_t i = 0; i < tensor_len; ++i) {
+    float diff = predict[i] - target[i];
+    if (diff < zero) {
+      diff = -diff;
+    }
+    if (diff < beta) {
+      out[i] = half * diff * diff / beta;
+    } else {
+      out[i] = diff - (half * beta);
+    }
+  }
+
+  return RET_OK;
+}
+
+int SmoothL1LossRun(void *cdata, int task_id) {
+  auto smooth_l1_loss_kernel = reinterpret_cast<SmoothL1LossCPUKernel *>(cdata);
+  auto error_code = smooth_l1_loss_kernel->Execute(task_id);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SmoothL1Loss error task_id[" << task_id << "] error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SmoothL1LossCPUKernel::Run() {
+  int error_code = ParallelLaunch(this->context_->thread_pool_, SmoothL1LossRun, this, 1);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SmoothL1Loss function error error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SmoothL1LossCPUKernel::Init() { return RET_OK; }
+
+kernel::LiteKernel *CpuSmoothL1LossFp32KernelCreator(const std::vector<lite::Tensor *> &inputs,
+                                                     const std::vector<lite::Tensor *> &outputs,
+                                                     OpParameter *opParameter, const lite::InnerContext *ctx,
+                                                     const kernel::KernelKey &desc,
+                                                     const mindspore::lite::PrimitiveC *primitive) {
+  MS_ASSERT(opParameter != nullptr);
+  MS_ASSERT(desc.type == schema::PrimitiveType_SmoothL1Loss);
+  auto *kernel = new (std::nothrow) SmoothL1LossCPUKernel(opParameter, inputs, outputs, ctx, primitive);
+  if (kernel == nullptr) {
+    MS_LOG(ERROR) << "new SmoothL1Loss failed";
+    return nullptr;
+  }
+  auto ret = kernel->Init();
+  if (ret != RET_OK) {
+    MS_LOG(ERROR) << "Init kernel failed, name: " << opParameter->name_ << ", type: "
+                  << schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(opParameter->type_));
+    delete kernel;
+    return nullptr;
+  }
+  return kernel;
+}
+
+REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_SmoothL1Loss, CpuSmoothL1LossFp32KernelCreator)
+}  // namespace mindspore::kernel

mindspore/lite/src/runtime/kernel/arm/fp32_grad/smooth_l1_loss.h

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SMOOTH_L1_LOSS_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SMOOTH_L1_LOSS_H_
+
+#include <vector>
+#include "src/lite_kernel.h"
+#include "nnacl/fp32_grad/smooth_l1_loss.h"
+
+namespace mindspore::kernel {
+class SmoothL1LossCPUKernel : public LiteKernel {
+ public:
+  explicit SmoothL1LossCPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
+                                 const std::vector<lite::Tensor *> &outputs, const lite::InnerContext *ctx,
+                                 const mindspore::lite::PrimitiveC *primitive)
+      : LiteKernel(parameter, inputs, outputs, ctx, primitive), smooth_l1_param_(nullptr) {
+    smooth_l1_param_ = reinterpret_cast<SmoothL1LossParameter *>(parameter);
+  }
+  ~SmoothL1LossCPUKernel() override {}
+  int Init() override;
+  int ReSize() override;
+  int Run() override;
+  int Execute(int task_id);
+
+ private:
+  SmoothL1LossParameter *smooth_l1_param_;
+};
+}  // namespace mindspore::kernel
+
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SMOOTH_L1_LOSS_H_

mindspore/lite/src/runtime/kernel/arm/fp32_grad/smooth_l1_loss_grad.cc

@@ -0,0 +1,99 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/runtime/kernel/arm/fp32_grad/smooth_l1_loss_grad.h"
+#include "src/kernel_registry.h"
+#include "include/errorcode.h"
+#include "src/runtime/runtime_api.h"
+
+using mindspore::lite::KernelRegistrar;
+using mindspore::lite::RET_ERROR;
+using mindspore::lite::RET_OK;
+using mindspore::schema::PrimitiveType_SmoothL1LossGrad;
+
+namespace mindspore::kernel {
+
+int SmoothL1LossGradCPUKernel::ReSize() { return RET_OK; }
+
+int SmoothL1LossGradCPUKernel::Execute(int task_id) {
+  SmoothL1LossParameter *smooth_l1_loss_param = reinterpret_cast<SmoothL1LossParameter *>(op_parameter_);
+
+  auto predict = reinterpret_cast<float *>(in_tensors_.at(0)->MutableData());
+  auto target = reinterpret_cast<float *>(in_tensors_.at(1)->MutableData());
+  auto d_loss = reinterpret_cast<float *>(in_tensors_.at(2)->MutableData());
+  auto *out = reinterpret_cast<float *>(out_tensors_.at(0)->MutableData());
+
+  const size_t tensor_len = in_tensors_.at(0)->ElementsNum();
+  const float beta = smooth_l1_loss_param->beta_;
+
+  for (uint64_t i = 0; i < tensor_len; ++i) {
+    float diff = predict[i] - target[i];
+    if (diff > beta) {
+      out[i] = d_loss[i];
+    } else if (diff < -beta) {
+      out[i] = -d_loss[i];
+    } else {
+      out[i] = (diff / beta) * d_loss[i];
+    }
+  }
+  return RET_OK;
+}
+
+int SmoothL1LossGradRun(void *cdata, int task_id) {
+  auto smooth_l1_loss_kernel = reinterpret_cast<SmoothL1LossGradCPUKernel *>(cdata);
+  auto error_code = smooth_l1_loss_kernel->Execute(task_id);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SmoothL1LossGrad error task_id[" << task_id << "] error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SmoothL1LossGradCPUKernel::Run() {
+  int error_code = ParallelLaunch(this->context_->thread_pool_, SmoothL1LossGradRun, this, 1);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "SmoothL1LossGrad function error error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int SmoothL1LossGradCPUKernel::Init() { return RET_OK; }
+
+kernel::LiteKernel *CpuSmoothL1LossGradFp32KernelCreator(const std::vector<lite::Tensor *> &inputs,
+                                                         const std::vector<lite::Tensor *> &outputs,
+                                                         OpParameter *opParameter, const lite::InnerContext *ctx,
+                                                         const kernel::KernelKey &desc,
+                                                         const mindspore::lite::PrimitiveC *primitive) {
+  MS_ASSERT(opParameter != nullptr);
+  MS_ASSERT(desc.type == schema::PrimitiveType_SmoothL1LossGrad);
+  auto *kernel = new (std::nothrow) SmoothL1LossGradCPUKernel(opParameter, inputs, outputs, ctx, primitive);
+  if (kernel == nullptr) {
+    MS_LOG(ERROR) << "new SmoothL1LossGradWithLogitsCPUKernel failed";
+    return nullptr;
+  }
+  auto ret = kernel->Init();
+  if (ret != RET_OK) {
+    MS_LOG(ERROR) << "Init kernel failed, name: " << opParameter->name_ << ", type: "
+                  << schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(opParameter->type_));
+    delete kernel;
+    return nullptr;
+  }
+  return kernel;
+}
+
+REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_SmoothL1LossGrad, CpuSmoothL1LossGradFp32KernelCreator)
+}  // namespace mindspore::kernel

mindspore/lite/src/runtime/kernel/arm/fp32_grad/smooth_l1_loss_grad.h

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SMOOTH_L1_LOSS_GRAD_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SMOOTH_L1_LOSS_GRAD_H_
+
+#include <vector>
+#include "src/lite_kernel.h"
+#include "nnacl/fp32_grad/smooth_l1_loss.h"
+
+namespace mindspore::kernel {
+class SmoothL1LossGradCPUKernel : public LiteKernel {
+ public:
+  explicit SmoothL1LossGradCPUKernel(OpParameter *parameter, const std::vector<lite::Tensor *> &inputs,
+                                     const std::vector<lite::Tensor *> &outputs, const lite::InnerContext *ctx,
+                                     const mindspore::lite::PrimitiveC *primitive)
+      : LiteKernel(parameter, inputs, outputs, ctx, primitive), smooth_l1_param_(nullptr) {
+    smooth_l1_param_ = reinterpret_cast<SmoothL1LossParameter *>(parameter);
+  }
+  ~SmoothL1LossGradCPUKernel() override {}
+  int Init() override;
+  int ReSize() override;
+  int Run() override;
+  int Execute(int task_id);
+
+ private:
+  SmoothL1LossParameter *smooth_l1_param_;
+};
+}  // namespace mindspore::kernel
+
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_GRAD_SMOOTH_L1_LOSS_GRAD_H_

mindspore/lite/src/train/train_populate_parameter.cc

@@ -44,7 +44,10 @@
 #include "src/ops/oneslike.h"
 #include "src/ops/binary_cross_entropy.h"
 #include "src/ops/binary_cross_entropy_grad.h"
-
+#include "src/ops/smooth_l1_loss.h"
+#include "src/ops/smooth_l1_loss_grad.h"
+#include "nnacl/fp32_grad/smooth_l1_loss.h"
+#include "src/ops/arithmetic_grad.h"
 namespace mindspore::kernel {
 
 OpParameter *DefaultPopulateParameter(const mindspore::lite::PrimitiveC *primitive) {
@@ -63,6 +66,44 @@ OpParameter *DefaultPopulateParameter(const mindspore::lite::PrimitiveC *primiti
   return param;
 }
 
+OpParameter *PopulateSmoothL1LossParameter(const mindspore::lite::PrimitiveC *primitive) {
+  if (primitive == nullptr) {
+    MS_LOG(ERROR) << "Primitive is nullptr when populating parameter for op.";
+    return nullptr;
+  }
+  SmoothL1LossParameter *p = reinterpret_cast<SmoothL1LossParameter *>(malloc(sizeof(SmoothL1LossParameter)));
+  if (p == nullptr) {
+    MS_LOG(ERROR) << "malloc SmoothL1LossParameter failed.";
+    return nullptr;
+  }
+  p->op_parameter_.type_ = primitive->Type();
+
+  auto smooth_l1_primitive =
+    reinterpret_cast<mindspore::lite::SmoothL1Loss *>(const_cast<mindspore::lite::PrimitiveC *>(primitive));
+
+  p->beta_ = smooth_l1_primitive->GetBeta();
+  return reinterpret_cast<OpParameter *>(p);
+}
+
+OpParameter *PopulateSmoothL1LossGradParameter(const mindspore::lite::PrimitiveC *primitive) {
+  if (primitive == nullptr) {
+    MS_LOG(ERROR) << "Primitive is nullptr when populating parameter for op.";
+    return nullptr;
+  }
+  SmoothL1LossParameter *p = reinterpret_cast<SmoothL1LossParameter *>(malloc(sizeof(SmoothL1LossParameter)));
+  if (p == nullptr) {
+    MS_LOG(ERROR) << "malloc SmoothL1LossParameter failed.";
+    return nullptr;
+  }
+  p->op_parameter_.type_ = primitive->Type();
+
+  auto smooth_l1_primitive =
+    reinterpret_cast<mindspore::lite::SmoothL1LossGrad *>(const_cast<mindspore::lite::PrimitiveC *>(primitive));
+
+  p->beta_ = smooth_l1_primitive->GetBeta();
+  return reinterpret_cast<OpParameter *>(p);
+}
+
 OpParameter *PopulateApplyMomentumParameter(const mindspore::lite::PrimitiveC *primitive) {
   if (primitive == nullptr) {
     MS_LOG(ERROR) << "Primitive is nullptr when populating parameter for op.";
@@ -473,14 +514,14 @@ OpParameter *PopulateArithmeticGradParameter(const mindspore::lite::PrimitiveC *
   }
   memset(arithmetic_param, 0, sizeof(ArithmeticParameter));
   arithmetic_param->op_parameter_.type_ = primitive->Type();
-  arithmetic_param->broadcasting_ = ((lite::Arithmetic *)primitive)->Broadcasting();
-  arithmetic_param->ndim_ = ((lite::Arithmetic *)primitive)->NDims();
+  arithmetic_param->broadcasting_ = ((lite::ArithmeticGrad *)primitive)->Broadcasting();
+  arithmetic_param->ndim_ = ((lite::ArithmeticGrad *)primitive)->NDims();
 
-  auto tmp_shape = ((lite::Arithmetic *)primitive)->InShape0();
+  auto tmp_shape = ((lite::ArithmeticGrad *)primitive)->x1Shape();
   memcpy(arithmetic_param->in_shape0_, static_cast<void *>(tmp_shape.data()), tmp_shape.size() * sizeof(int));
-  tmp_shape = ((lite::Arithmetic *)primitive)->InShape1();
+  tmp_shape = ((lite::ArithmeticGrad *)primitive)->x2Shape();
   memcpy(arithmetic_param->in_shape1_, static_cast<void *>(tmp_shape.data()), tmp_shape.size() * sizeof(int));
-  tmp_shape = ((lite::Arithmetic *)primitive)->OutputShape();
+  tmp_shape = ((lite::ArithmeticGrad *)primitive)->dyShape();
   memcpy(arithmetic_param->out_shape_, static_cast<void *>(tmp_shape.data()), tmp_shape.size() * sizeof(int));
   return reinterpret_cast<OpParameter *>(arithmetic_param);
 }
@@ -518,6 +559,12 @@ void PopulateTrainParameters() {
   lite::Registry DropGradParameterRegistry(schema::PrimitiveType_DropoutGrad, PopulateDropoutGradParameter);
   lite::Registry MaximumGradParameterRegistry(schema::PrimitiveType_MaximumGrad, PopulateArithmeticGradParameter);
   lite::Registry MinimumGradParameterRegistry(schema::PrimitiveType_MinimumGrad, PopulateArithmeticGradParameter);
+  lite::Registry SmoothL1LossRegistry(schema::PrimitiveType_SmoothL1Loss, PopulateSmoothL1LossParameter);
+  lite::Registry SmoothL1LossGradRegistry(schema::PrimitiveType_SmoothL1LossGrad, PopulateSmoothL1LossGradParameter);
+  lite::Registry SigmoidCrossEntropyWithLogitsRegistry(schema::PrimitiveType_SigmoidCrossEntropyWithLogits,
+                                                       DefaultPopulateParameter);
+  lite::Registry SigmoidCrossEntropyWithLogitsGradRegistry(schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad,
+                                                           DefaultPopulateParameter);
 }
 
 }  // namespace mindspore::kernel

mindspore/lite/src/train/train_session.cc

@@ -304,7 +304,11 @@ void TrainSession::CompileOptimizedKernels() {
 
 bool TrainSession::IsLossKernel(const kernel::LiteKernel *kernel) const {
   return (kernel->Type() == schema::PrimitiveType_SoftmaxCrossEntropy ||
-          kernel->Type() == schema::PrimitiveType_SparseSoftmaxCrossEntropy);
+          kernel->Type() == schema::PrimitiveType_SparseSoftmaxCrossEntropy ||
+          kernel->Type() == schema::PrimitiveType_SmoothL1Loss ||
+          kernel->Type() == schema::PrimitiveType_SmoothL1LossGrad ||
+          kernel->Type() == schema::PrimitiveType_SigmoidCrossEntropyWithLogits ||
+          kernel->Type() == schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad);
 }
 
 bool TrainSession::IsOptimizer(kernel::LiteKernel *kernel) const {

mindspore/lite/test/models_ms_train.cfg

@@ -3,6 +3,9 @@ mini_alexnet
 mobilenetv2
 mobilenetv3
 lenet
-#effnet
+effnet
 effnet_tune
-#resnet
+# lenetv1
+# resnet
+# effnetv1
+#LAST

mindspore/lite/test/run_net_train.sh

@@ -15,8 +15,8 @@ function Run_Export(){
         fi
         echo ${model_name}'_train_export.py' >> "${export_log_file}"
         echo 'exporting' ${model_name}
-        echo 'docker run --user $(id -u):$(id -g) --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore_dev:5 python '${models_path}'/'${model_name}'_train_export.py' >>  "${export_log_file}"
-        docker run --user $(id -u):$(id -g) --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore_dev:5 python ${models_path}'/'${model_name}_train_export.py ${epoch_num}
+        echo 'docker run --user "$(id -u):$(id -g)" --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore/mindspore-gpu:1.0.0 python '${models_path}'/'${model_name}'_train_export.py' >>  "${export_log_file}"
+        docker run --user "$(id -u):$(id -g)" --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore/mindspore-gpu:1.0.0 python ${models_path}'/'${model_name}_train_export.py "${epoch_num}"
         if [ $? = 0 ]; then
             export_result='export mindspore '${model_name}'_train_export pass';echo ${export_result} >> ${export_result_file}
         else
@@ -74,7 +74,7 @@ function Run_x86() {
         echo ${model_name}'_train' >> "${run_x86_log_file}"
         echo 'cd  '${x86_path}'/mindspore-lite-'${version}'-runtime-x86-'${process_unit_x86}-train >> "${run_x86_log_file}"
         cd ${x86_path}/mindspore-lite-${version}-runtime-x86-${process_unit_x86}-train || return 1
-        echo 'LD_LIBRARY_PATH='${LD_LIBRARY_PATH}':./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./net_train/net_train --epochs='${epoch_num}' --modelFile='${ms_models_path}'/'${model_name}'_train.ms --inDataFile='${input_path}'/'${model_name}'_input1.bin,'${train_io_path}'/'${model_name}'_input2.bin --expectedDataFile='${train_io_path}'/'${model_name}'_outputs.bin --exportFile='${ms_models_path}'/'${model_name}'_train_exported.ms'  >> "${run_x86_log_file}"
+        echo 'LD_LIBRARY_PATH='${LD_LIBRARY_PATH}':./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib;./net_train/net_train --epochs='${epoch_num}' --modelFile='${ms_models_path}'/'${model_name}'_train.ms --inDataFile='${train_io_path}/${model_name}_input1.bin,${train_io_path}/${model_name}_input2.bin' --expectedDataFile='${train_io_path}'/'${model_name}'_outputs.bin --exportFile='${ms_models_path}'/'${model_name}'_train_exported.ms'  >> "${run_x86_log_file}"
         echo '-------------------------------------------------------------------------------' >> "${run_x86_log_file}"
         LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./lib:./third_party/libjpeg-turbo/lib:./third_party/opencv/lib \
         ${run_valgrind}./net_train/net_train \
@@ -94,6 +94,7 @@ function Run_x86() {
 # Run on arm platform: 
 # Gets a parameter - arm64/arm32
 function Run_arm() {
+    tmp_dir=/data/local/tmp/net_train_test
     if [ "$1" == arm64 ]; then
         arm_path=${arm64_path}
         process_unit=${process_unit_arm64}
@@ -161,14 +162,22 @@ function Run_arm() {
         echo 'chmod 777 net_train' >> ${adb_cmd_run_file}
         if [ "$1" == arm64 ]; then
             echo 'cp  /data/local/tmp/libc++_shared.so ./' >> ${adb_cmd_run_file}
-            echo 'export LD_LIBRARY_PATH=./:/data/local/tmp/net_train_test;./net_train --epochs='${epoch_num}' --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin' >> "${run_arm_log_file}"
-            echo 'export LD_LIBRARY_PATH=./:/data/local/tmp/net_train_test;./net_train --epochs='${epoch_num}' --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin' >> "${adb_cmd_run_file}"
         elif [ "$1" == arm32 ]; then
             echo 'cp  /data/local/tmp/arm32/libc++_shared.so ./' >> ${adb_cmd_run_file}
-            echo 'export LD_LIBRARY_PATH=./:/data/local/tmp/:/data/local/tmp/net_train_test;./net_train --epochs='${epoch_num}' --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin'  >> "${run_arm_log_file}"
-            echo 'export LD_LIBRARY_PATH=./:/data/local/tmp/:/data/local/tmp/net_train_test;./net_train --epochs='${epoch_num}' --modelFile='${model_name}'_train.ms --inDataFile=/data/local/tmp/net_train_test/'${model_name}'_input1.bin,/data/local/tmp/net_train_test/'${model_name}'_input2.bin --expectedDataFile=/data/local/tmp/net_train_test/'${model_name}'_outputs.bin'  >> "${adb_cmd_run_file}"
-        fi    
-
+        fi 
+        echo "rm -f ${tmp_dir}/${model_name}_train_exported.ms" >> ${run_arm_log_file}
+        echo "rm -f ${tmp_dir}/${model_name}_train_exported.ms" >> ${adb_cmd_run_file}
+        adb_cmd=$(cat <<-ENDM
+        export LD_LIBRARY_PATH=./:/data/local/tmp/:/data/local/tmp/net_train_test;./net_train \
+        --epochs=${epoch_num} \
+        --modelFile=${model_name}_train.ms \
+        --inDataFile=${tmp_dir}/${model_name}_input1.bin,${tmp_dir}/${model_name}_input2.bin \
+        --expectedDataFile=${tmp_dir}/${model_name}_outputs.bin \
+        --exportFile=${tmp_dir}/${model_name}_train_exported.ms 
+ENDM
+        )
+        echo "${adb_cmd}" >> ${run_arm_log_file}
+        echo "${adb_cmd}" >> ${adb_cmd_run_file}
         adb -s ${device_id} shell < ${adb_cmd_run_file} >> ${run_arm_log_file}
         # TODO: change to arm_type
         if [ $? = 0 ]; then
@@ -340,7 +349,7 @@ adb_cmd_arm64_file=${logs_path}/adb_arm64_cmd.txt
 adb_cmd_arm64_run_file=${logs_path}/adb_arm64_cmd_run.txt
 
 run_arm32_log_file=${logs_path}/run_arm32_log.txt
-echo 'run arm32 logs: ' > ${run_arm64_log_file}
+echo 'run arm32 logs: ' > ${run_arm32_log_file}
 adb_push_arm32_log_file=${logs_path}/adb_push_arm32_log.txt
 adb_cmd_arm32_file=${logs_path}/adb_arm32_cmd.txt
 adb_cmd_arm32_run_file=${logs_path}/adb_arm32_cmd_run.txt
@@ -358,11 +367,11 @@ Run_x86 &
 Run_x86_PID=$!
 sleep 1
 
+
 # wait ${Run_x86_PID}
 cat ${run_net_train_result_file}
 wait ${Run_x86_PID}
 Run_x86_status=$?
-# exit 0
 
 # Run on arm64
 echo "start Run arm64 ..."

mindspore/lite/tools/common/node_util.cc

@@ -137,11 +137,10 @@ static const std::vector<schema::PrimitiveType> int8OpList = {schema::PrimitiveT
 
 static const std::vector<schema::PrimitiveType> needInsertOpList = {
 #ifdef SUPPORT_TRAIN
-  schema::PrimitiveType_Eltwise,      schema::PrimitiveType_Activation,
-  schema::PrimitiveType_Concat,       schema::PrimitiveType_Power,
-  schema::PrimitiveType_StridedSlice, schema::PrimitiveType_Add,
-  schema::PrimitiveType_Split,        schema::PrimitiveType_Slice,
-  schema::PrimitiveType_Crop
+  schema::PrimitiveType_Eltwise, schema::PrimitiveType_Activation,   schema::PrimitiveType_Concat,
+  schema::PrimitiveType_Power,   schema::PrimitiveType_StridedSlice, schema::PrimitiveType_Split,
+  schema::PrimitiveType_Slice,   schema::PrimitiveType_Crop,         schema::PrimitiveType_Mul,
+  schema::PrimitiveType_Add
 #else
   schema::PrimitiveType_Eltwise, schema::PrimitiveType_Activation,   schema::PrimitiveType_Concat,
   schema::PrimitiveType_Power,   schema::PrimitiveType_StridedSlice, schema::PrimitiveType_Add,

mindspore/lite/tools/converter/legacy_optimizer/graph/trans_format_insert_pass.cc

@@ -171,6 +171,13 @@ STATUS TransOpInsertPass::Run(schema::MetaGraphT *graph) {
       STATUS status = RET_OK;
       auto input_tensor_size = (*iter)->inputIndex.size();
       for (size_t i = 0; i < input_tensor_size; i++) {
+#ifdef SUPPORT_TRAIN
+        auto &tensor = graph->allTensors.at((*iter)->inputIndex[i]);
+        MS_ASSERT(tensor != nullptr);
+        if (tensor->nodeType == schema::NodeType_ValueNode) {
+          continue;
+        }
+#endif
         iter = InsertFormatTransNode(graph, iter, kBefore, i, pre_insert_trans_type_, &status);
         if (status != RET_OK) {
           MS_LOG(ERROR) << "Insert" << pre_insert_trans_type_ << "before " << (*iter)->name << " failed";