!10119 tod qa fixes

From: @yonibaehr_admin
Reviewed-by: @HilbertDavid,@ddwsky
Signed-off-by: @HilbertDavid

mindspore/lite/examples/train_lenet/model/lenet_export.py

@@ -14,11 +14,11 @@
 # ============================================================================
 """lenet_export."""
 
+import numpy as np
 from mindspore import context, Tensor
 import mindspore.common.dtype as mstype
 from mindspore.train.serialization import export
 from lenet import LeNet5
-import numpy as np
 from train_utils import TrainWrap
 
 
@@ -30,6 +30,6 @@ BATCH_SIZE = 32
 x = Tensor(np.ones((BATCH_SIZE, 1, 32, 32)), mstype.float32)
 label = Tensor(np.zeros([BATCH_SIZE, 10]).astype(np.float32))
 net = TrainWrap(n)
-export(net, x, label, file_name="lenet_tod.mindir", file_format='MINDIR')
+export(net, x, label, file_name="lenet_tod", file_format='MINDIR')
 
 print("finished exporting")

mindspore/lite/examples/train_lenet/model/prepare_model.sh

@@ -14,7 +14,7 @@ CONVERTER="../../../build/tools/converter/converter_lite"
 if [ ! -f "$CONVERTER" ]; then
   if ! command -v converter_lite &> /dev/null
   then
-    tar -xzf ../../../../../output/mindspore-lite-*-converter-ubuntu-train.tar.gz --strip-components 2 --wildcards --no-anchored converter_lite libmindspore_gvar.so
+    tar -xzf ../../../../../output/mindspore-lite-*-train-converter-linux-x64.tar.gz --strip-components 2 --wildcards --no-anchored converter_lite libmindspore_gvar.so
     if [ -f ./converter_lite ]; then
       CONVERTER=./converter_lite
     else

mindspore/lite/examples/train_lenet/prepare_and_run.sh

@@ -47,7 +47,11 @@ if [ "$MNIST_DATA_PATH" == "" ]; then
 fi
 
 if [ "$TARBALL" == "" ]; then
-  file=$(ls ../../../../output/mindspore-lite-*-runtime-${TARGET}-cpu-train.tar.gz)
+  if [ "${TARGET}" == "arm64" ]; then
+    file=$(ls ../../../../output/mindspore-lite-*-train-android-aarch64.tar.gz)
+  else
+    file=$(ls ../../../../output/mindspore-lite-*-train-linux-x64.tar.gz)
+  fi
   if [ -f ${file} ]; then
     TARBALL=${file}
   else

mindspore/lite/examples/transfer_learning/Makefile

@@ -35,7 +35,7 @@ clean:
 
 
 mrproper: 
-	rm -rf dataset package* msl src/*.o bin/ model/*.mindir model/*.ms 
+	rm -rf dataset package* msl src/*.o bin/ model/*.mindir model/*.ms model/*.so model/converter_lite
 
 %.o:%.cc
 	$(CXX) $(CFLAGS) -c $< -o $@

mindspore/lite/examples/transfer_learning/model/prepare_model.sh

@@ -18,7 +18,7 @@ CONVERTER="../../../build/tools/converter/converter_lite"
 if [ ! -f "$CONVERTER" ]; then
   if ! command -v converter_lite &> /dev/null
   then
-    tar -xzf ../../../../../output/mindspore-lite-*-converter-ubuntu-train.tar.gz --strip-components 2 --wildcards --no-anchored converter_lite libmindspore_gvar.so
+    tar -xzf ../../../../../output/mindspore-lite-*-train-converter-linux-x64.tar.gz --strip-components 2 --wildcards --no-anchored converter_lite libmindspore_gvar.so
     if [ -f ./converter_lite ]; then
       CONVERTER=./converter_lite
     else

mindspore/lite/examples/transfer_learning/model/transfer_learning_export.py

@@ -14,13 +14,13 @@
 # ============================================================================
 """transfer_learning_export."""
 
+import numpy as np
 import mindspore as M
 from mindspore.nn import Cell
 from mindspore.train.serialization import load_checkpoint
 from mindspore.common.parameter import ParameterTuple
 from mindspore.train.serialization import export
 from effnet import effnet
-import numpy as np
 from train_utils import TrainWrap
 
 
@@ -51,13 +51,13 @@ trainable_weights = ParameterTuple(trainable_weights_list)
 
 M.context.set_context(mode=M.context.PYNATIVE_MODE,
                       device_target="GPU", save_graphs=False)
-BATCH_SIZE = 32
+BATCH_SIZE = 16
 X = M.Tensor(np.ones((BATCH_SIZE, 3, 224, 224)), M.float32)
 label = M.Tensor(np.zeros([BATCH_SIZE, 10]).astype(np.float32))
 
 sgd = M.nn.SGD(trainable_weights, learning_rate=0.01, momentum=0.9,
                dampening=0.01, weight_decay=0.0, nesterov=False, loss_scale=1.0)
 net = TrainWrap(n, optimizer=sgd, weights=trainable_weights)
-export(net, X, label, file_name="transfer_learning_tod.mindir", file_format='MINDIR')
+export(net, X, label, file_name="transfer_learning_tod", file_format='MINDIR')
 
 print("Exported")

mindspore/lite/examples/transfer_learning/prepare_and_run.sh

@@ -46,7 +46,11 @@ if [ "$PLACES_DATA_PATH" == "" ]; then
 fi
 
 if [ "$TARBALL" == "" ]; then
-  file=$(ls ../../../../output/mindspore-lite-*-runtime-${TARGET}-cpu-train.tar.gz)
+  if [ "${TARGET}" == "arm64" ]; then
+    file=$(ls ../../../../output/mindspore-lite-*-train-android-aarch64.tar.gz)
+  else
+    file=$(ls ../../../../output/mindspore-lite-*-train-linux-x64.tar.gz)
+  fi
   if [ -f ${file} ]; then
     TARBALL=${file}
   else
@@ -83,7 +87,7 @@ mv mindspore-*/* msl/
 rm -rf mindspore-*
 
 # Convert the dataset into the package
-./prepare_dataset.sh ${PLACES_DATA_PATH}
+./prepare_dataset.sh ${PLACES_DATA_PATH} || exit 1
 cp -r dataset ${PACKAGE}
 
 echo "==========Compiling============"

mindspore/lite/examples/transfer_learning/prepare_dataset.sh

@@ -1,19 +1,27 @@
 #!/bin/bash
 
 echo "=======Preparing Dataset======="
-[ -d "dataset" ] && echo "dataset was already created" && exit
-echo "Preparing dataset" 
+[ -d "dataset" ] && echo "dataset was already created" && exit 0
 PLACES_DATA_PATH=$1
+if [ ! -d ${PLACES_DATA_PATH}/val_256/ ]; then
+  echo "The path" ${PLACES_DATA_PATH} "does not contain Places validation dataset. Please read the README file!" && exit 1
+fi
 class_id=0
+sp="/-\|"
 classes=("4" "98" "6" "7" "10" "15" "17" "70" "26" "30")
+echo -n 'Prep class '
 for class in "${classes[@]}"; do
   mkdir -p dataset/$class_id
-  i=0
+  f=0
+  i=1
+  echo -n $(($class_id+1)) ' '
   cat scripts/places365_val.txt | grep -w ${class} | awk '{print $1}' | while read line
   do 
-    echo converting ${PLACES_DATA_PATH}/val_256/$line to bmp
-    convert -colorspace RGB -gravity center -crop '224x224+0+0' ${PLACES_DATA_PATH}/val_256/$line dataset/$class_id/$i.bmp; 
-    i=$(($i+1)); 
+    printf "\b${sp:i++%${#sp}:1}"
+    convert -colorspace RGB -gravity center -crop '224x224+0+0' ${PLACES_DATA_PATH}/val_256/$line dataset/$class_id/$f.bmp; 
+    f=$(($f+1)); 
   done
+  printf "\b"
   class_id=$(($class_id+1))
 done
+echo ' '

mindspore/lite/examples/transfer_learning/scripts/train.sh

@@ -14,4 +14,4 @@
 # limitations under the License.
 # ============================================================================
 
-LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/transfer_learning_tod.ms -e 60 -d dataset -s 20
+LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/transfer_learning_tod.ms -e 100 -d dataset -s 20

mindspore/lite/test/run_net_train.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # Run Export on x86 platform and create output test files:
+docker_image=mindspore_dev:8
 function Run_Export(){
     cd $models_path || exit 1
     if [[ -z "${CLOUD_MODEL_ZOO}" ]]; then
@@ -15,8 +16,8 @@ function Run_Export(){
         fi
         echo ${model_name}'_train_export.py' >> "${export_log_file}"
         echo 'exporting' ${model_name}
-        echo 'docker run --user "$(id -u):$(id -g)" --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore/mindspore-gpu:1.0.0 python '${models_path}'/'${model_name}'_train_export.py' >>  "${export_log_file}"
-        docker run --user "$(id -u):$(id -g)" --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true mindspore/mindspore-gpu:1.0.0 python ${models_path}'/'${model_name}_train_export.py "${epoch_num}"
+        echo 'docker run --user '"$(id -u):$(id -g)"' --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true '${docker_image}' python '${models_path}'/'${model_name}'_train_export.py' >>  "${export_log_file}"
+        docker run --user "$(id -u):$(id -g)" --env CLOUD_MODEL_ZOO=${CLOUD_MODEL_ZOO} -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER -v /opt/share:/opt/share --privileged=true "${docker_image}" python ${models_path}'/'${model_name}_train_export.py "${epoch_num}"
         if [ $? = 0 ]; then
             export_result='export mindspore '${model_name}'_train_export pass';echo ${export_result} >> ${export_result_file}
         else
@@ -219,32 +220,32 @@ train_io_path=""
 while getopts "r:m:d:i:e:vt:" opt; do
     case ${opt} in
         r)
-	    release_path=${OPTARG}
-            echo "release_path is ${OPTARG}"
+           release_path=${OPTARG}
+           echo "release_path is ${OPTARG}"
             ;;
         m)
 
-	    models_path=${OPTARG}"/models_train"
+            models_path=${OPTARG}"/models_train"
             echo "models_path is ${OPTARG}"
             ;;
         i)
-	    train_io_path=${OPTARG}
+            train_io_path=${OPTARG}
             echo "train_io_path is ${OPTARG}"
             ;;
         d)
-	    device_id=${OPTARG}
+           device_id=${OPTARG}
             echo "device_id is ${OPTARG}"
             ;;
         e)
-	    enable_export=${OPTARG}
+            enable_export=${OPTARG}
             echo "enable_export = ${OPTARG}"
             ;;          
         v)
-	    run_valgrind="valgrind --log-file=valgrind.log "
+            run_valgrind="valgrind --log-file=valgrind.log "
             echo "Run x86 with valgrind"
             ;;
         t)
-	    epoch_num=${OPTARG}
+            epoch_num=${OPTARG}
             echo "train epoch num is ${OPTARG}"
             ;;                          
         ?)

mindspore/lite/tools/anf_exporter/anf_exporter.cc

@@ -429,6 +429,7 @@ int AnfExporter::ConvertInputValueNode(const std::shared_ptr<AnfNode> &input_ano
     paramTensor->dims = dims;
 #ifdef SUPPORT_TRAIN
     if (paramTensor->dims.size() == 0) paramTensor->dims = {1};
+    paramTensor->name = valueNode->fullname_with_scope();
 #endif
     paramTensor->nodeType = schema::NodeType::NodeType_ValueNode;
     auto data = value->cast<tensor::TensorPtr>();

mindspore/lite/tools/anf_importer/import_from_mindir.cc

@@ -855,6 +855,14 @@ int AnfImporterFromMindir::ParseModelConfigureInfo(const onnx::ModelProto &model
 }
 
 int AnfImporterFromMindir::Import(const converter::Flags *flag) {
+#if SUPPORT_TRAIN
+  func_graph_ = LoadMindIR(flag->modelFile);
+  if (func_graph_ != nullptr) {
+    return RET_OK;
+  } else {
+    MS_LOG(ERROR) << "Parse new mind_ir proto failed, Trying old onnx format";
+  }
+#endif
   onnx_model_ = ReadOnnxFromBinary(flag->modelFile);
   if (onnx_model_ == nullptr) {
     MS_LOG(DEBUG) << "Parse model failed, which is not an old mindir model";

mindspore/lite/tools/net_train/net_train.cc

@@ -679,7 +679,7 @@ NetTrain::~NetTrain() {
     delete (iter.second);
   }
   this->data_.clear();
-  delete (session_);
+  if (session_ != nullptr) delete (session_);
 }
 
 int RunNetTrain(int argc, const char **argv) {

mindspore/lite/tools/net_train/net_train.h

@@ -219,7 +219,7 @@ class MS_API NetTrain {
 
  private:
   NetTrainFlags *flags_;
-  session::TrainSession *session_;
+  session::TrainSession *session_ = nullptr;
   std::vector<mindspore::tensor::MSTensor *> ms_inputs_;
   std::unordered_map<std::string, std::vector<mindspore::tensor::MSTensor *>> ms_outputs_;
   std::unordered_map<std::string, CheckTensor *> data_;