From 3e7df2f69b76b6de9ed5ebe51b6131628af90d8d Mon Sep 17 00:00:00 2001 From: zhaoting Date: Mon, 19 Apr 2021 10:09:01 +0800 Subject: [PATCH] clean pylint & shell check --- .../official/cv/crnn/ascend310_infer/build.sh | 2 +- .../official/cv/crnn/scripts/run_infer_310.sh | 4 +- .../official/cv/ctpn/scripts/run_infer_310.sh | 2 +- .../cv/deeptext/ascend310_infer/build.sh | 2 +- model_zoo/official/cv/deeptext/postprocess.py | 5 +- .../cv/deeptext/scripts/run_infer_310.sh | 4 +- .../faster_rcnn/ascend310_infer/src/build.sh | 2 +- .../cv/maskrcnn/scripts/run_infer_310.sh | 4 +- .../official/cv/resnet152/scripts/run_eval.sh | 2 +- .../official/cv/unet/scripts/run_infer_310.sh | 2 +- .../cv/vgg16/scripts/run_distribute_train.sh | 3 +- .../cv/yolov4/scripts/run_infer_310.sh | 2 +- .../official/nlp/fasttext/src/dataset.py | 59 ++++++++++--------- .../FaceQualityAssessment/scripts/run_eval.sh | 2 +- .../cv/FaceQualityAssessment/train.py | 2 +- model_zoo/research/cv/FaceRecognition/eval.py | 2 +- .../scripts/run_standalone_train_base.sh | 2 +- .../scripts/run_standalone_train_beta.sh | 2 +- .../cv/FaceRecognitionForTracking/train.py | 8 +-- .../nlp/ternarybert/scripts/run_train.sh | 2 +- 20 files changed, 59 insertions(+), 54 deletions(-) diff --git a/model_zoo/official/cv/crnn/ascend310_infer/build.sh b/model_zoo/official/cv/crnn/ascend310_infer/build.sh index 4e1817bac6..90ce11f12d 100644 --- a/model_zoo/official/cv/crnn/ascend310_infer/build.sh +++ b/model_zoo/official/cv/crnn/ascend310_infer/build.sh @@ -17,7 +17,7 @@ if [ ! -d out ]; then mkdir out fi -cd out +cd out || exit cmake .. \ -DMINDSPORE_PATH="`pip show mindspore-ascend | grep Location | awk '{print $2"/mindspore"}' | xargs realpath`" make diff --git a/model_zoo/official/cv/crnn/scripts/run_infer_310.sh b/model_zoo/official/cv/crnn/scripts/run_infer_310.sh index fa64118a51..4efd4497a6 100755 --- a/model_zoo/official/cv/crnn/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/crnn/scripts/run_infer_310.sh @@ -63,7 +63,7 @@ fi function compile_app() { - cd ../ascend310_infer + cd ../ascend310_infer || exit if [ -f "Makefile" ]; then make clean fi @@ -73,7 +73,7 @@ function compile_app() echo "compile app code failed" exit 1 fi - cd - + cd - || exit } function infer() diff --git a/model_zoo/official/cv/ctpn/scripts/run_infer_310.sh b/model_zoo/official/cv/ctpn/scripts/run_infer_310.sh index dac4cbbf18..f42dafc1d1 100755 --- a/model_zoo/official/cv/ctpn/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/ctpn/scripts/run_infer_310.sh @@ -73,7 +73,7 @@ function compile_app() echo "compile app code failed" exit 1 fi - cd - + cd - || exit } function infer() diff --git a/model_zoo/official/cv/deeptext/ascend310_infer/build.sh b/model_zoo/official/cv/deeptext/ascend310_infer/build.sh index 8bf761bc33..770a8851ef 100644 --- a/model_zoo/official/cv/deeptext/ascend310_infer/build.sh +++ b/model_zoo/official/cv/deeptext/ascend310_infer/build.sh @@ -17,7 +17,7 @@ if [ ! -d out ]; then mkdir out fi -cd out +cd out || exit cmake .. \ -DMINDSPORE_PATH="`pip show mindspore-ascend | grep Location | awk '{print $2"/mindspore"}' | xargs realpath`" make diff --git a/model_zoo/official/cv/deeptext/postprocess.py b/model_zoo/official/cv/deeptext/postprocess.py index 55a0c41bd6..a8e3fd2c1e 100644 --- a/model_zoo/official/cv/deeptext/postprocess.py +++ b/model_zoo/official/cv/deeptext/postprocess.py @@ -16,12 +16,11 @@ """Evaluation for Deeptext""" import argparse import os - +from PIL import Image import numpy as np +import mmcv from src.config import config from src.utils import metrics -from PIL import Image -import mmcv parser = argparse.ArgumentParser(description="Deeptext evaluation") parser.add_argument("--result_path", type=str, required=True, help="result file path") diff --git a/model_zoo/official/cv/deeptext/scripts/run_infer_310.sh b/model_zoo/official/cv/deeptext/scripts/run_infer_310.sh index 5e9ea2ae4e..671d5d1378 100755 --- a/model_zoo/official/cv/deeptext/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/deeptext/scripts/run_infer_310.sh @@ -62,7 +62,7 @@ fi function compile_app() { - cd ../ascend310_infer + cd ../ascend310_infer || exit if [ -f "Makefile" ]; then make clean fi @@ -72,7 +72,7 @@ function compile_app() echo "compile app code failed" exit 1 fi - cd - + cd - || exit } function infer() diff --git a/model_zoo/official/cv/faster_rcnn/ascend310_infer/src/build.sh b/model_zoo/official/cv/faster_rcnn/ascend310_infer/src/build.sh index 82d02bf3f0..f00e5e91fa 100755 --- a/model_zoo/official/cv/faster_rcnn/ascend310_infer/src/build.sh +++ b/model_zoo/official/cv/faster_rcnn/ascend310_infer/src/build.sh @@ -19,7 +19,7 @@ build_type="Release" function preparePath() { rm -rf $1 mkdir -p $1 - cd $1 + cd $1 || exit } function buildA300() { diff --git a/model_zoo/official/cv/maskrcnn/scripts/run_infer_310.sh b/model_zoo/official/cv/maskrcnn/scripts/run_infer_310.sh index 9a93ffc656..f9ccce5c14 100755 --- a/model_zoo/official/cv/maskrcnn/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/maskrcnn/scripts/run_infer_310.sh @@ -66,9 +66,9 @@ function air_to_om() function compile_app() { - cd ../ascend310_infer/src + cd ../ascend310_infer/src || exit sh build.sh &> build.log - cd - + cd - || exit } function infer() diff --git a/model_zoo/official/cv/resnet152/scripts/run_eval.sh b/model_zoo/official/cv/resnet152/scripts/run_eval.sh index 1cc5c35828..6affcd4e93 100644 --- a/model_zoo/official/cv/resnet152/scripts/run_eval.sh +++ b/model_zoo/official/cv/resnet152/scripts/run_eval.sh @@ -57,7 +57,7 @@ mkdir ./eval cp ../*.py ./eval cp *.sh ./eval cp -r ../src ./eval -cd ./eval +cd ./eval || exit env > env.log echo "start evaluation for device $DEVICE_ID" python eval.py --data_url=$PATH1 --checkpoint_path=$PATH2 &> eval.log & diff --git a/model_zoo/official/cv/unet/scripts/run_infer_310.sh b/model_zoo/official/cv/unet/scripts/run_infer_310.sh index 7b4e54fa75..5e8253fd9f 100644 --- a/model_zoo/official/cv/unet/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/unet/scripts/run_infer_310.sh @@ -70,7 +70,7 @@ function preprocess_data() function compile_app() { - cd ../ascend310_infer/src + cd ../ascend310_infer/ || exit if [ -f "Makefile" ]; then make clean fi diff --git a/model_zoo/official/cv/vgg16/scripts/run_distribute_train.sh b/model_zoo/official/cv/vgg16/scripts/run_distribute_train.sh index 4b64e18ce2..ee3f061557 100644 --- a/model_zoo/official/cv/vgg16/scripts/run_distribute_train.sh +++ b/model_zoo/official/cv/vgg16/scripts/run_distribute_train.sh @@ -62,7 +62,8 @@ do end=`expr $start \+ $gap` cmdopt=$start"-"$end - export DEVICE_ID=`expr $i \+ $start_idx` + device_id=`expr $i \+ $start_idx` + export DEVICE_ID=$device_id export RANK_ID=$i rm -rf ./train_parallel$DEVICE_ID mkdir ./train_parallel$DEVICE_ID diff --git a/model_zoo/official/cv/yolov4/scripts/run_infer_310.sh b/model_zoo/official/cv/yolov4/scripts/run_infer_310.sh index c23c0c7146..16dc222982 100644 --- a/model_zoo/official/cv/yolov4/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/yolov4/scripts/run_infer_310.sh @@ -60,7 +60,7 @@ fi function compile_app() { - cd ../ascend310_infer/src + cd ../ascend310_infer/src || exit if [ -f "Makefile" ]; then make clean fi diff --git a/model_zoo/official/nlp/fasttext/src/dataset.py b/model_zoo/official/nlp/fasttext/src/dataset.py index 7135bd4590..0f66c912ea 100644 --- a/model_zoo/official/nlp/fasttext/src/dataset.py +++ b/model_zoo/official/nlp/fasttext/src/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ import pprint import ast import html import numpy as np -import spacy from sklearn.feature_extraction import FeatureHasher +import spacy from mindspore.mindrecord import FileWriter @@ -63,6 +63,33 @@ class FastTextDataPreProcess(): self.vec2words[1] = 'UNK' self.str_html = re.compile(r'<[^>]+>') + def common_block(self, _pair_sen, spacy_nlp): + """common block for data preprocessing""" + label_idx = int(_pair_sen[0]) - 1 + if len(_pair_sen) == 3: + src_tokens = self.input_preprocess(src_text1=_pair_sen[1], + src_text2=_pair_sen[2], + spacy_nlp=spacy_nlp, + train_mode=True) + src_tokens_length = len(src_tokens) + elif len(_pair_sen) == 2: + src_tokens = self.input_preprocess(src_text1=_pair_sen[1], + src_text2=None, + spacy_nlp=spacy_nlp, + train_mode=True) + src_tokens_length = len(src_tokens) + elif len(_pair_sen) == 4: + if _pair_sen[2]: + sen_o_t = _pair_sen[1] + ' ' + _pair_sen[2] + else: + sen_o_t = _pair_sen[1] + src_tokens = self.input_preprocess(src_text1=sen_o_t, + src_text2=_pair_sen[3], + spacy_nlp=spacy_nlp, + train_mode=True) + src_tokens_length = len(src_tokens) + return src_tokens, src_tokens_length, label_idx + def load(self): """data preprocess loader""" train_dataset_list = [] @@ -73,30 +100,8 @@ class FastTextDataPreProcess(): with open(self.train_path, 'r', newline='', encoding='utf-8') as src_file: reader = csv.reader(src_file, delimiter=",", quotechar='"') for _, _pair_sen in enumerate(reader): - label_idx = int(_pair_sen[0]) - 1 - if len(_pair_sen) == 3: - src_tokens = self.input_preprocess(src_text1=_pair_sen[1], - src_text2=_pair_sen[2], - spacy_nlp=spacy_nlp, - train_mode=True) - src_tokens_length = len(src_tokens) - elif len(_pair_sen) == 2: - src_tokens = self.input_preprocess(src_text1=_pair_sen[1], - src_text2=None, - spacy_nlp=spacy_nlp, - train_mode=True) - src_tokens_length = len(src_tokens) - elif len(_pair_sen) == 4: - if _pair_sen[2]: - sen_o_t = _pair_sen[1] + ' ' + _pair_sen[2] - else: - sen_o_t = _pair_sen[1] - src_tokens = self.input_preprocess(src_text1=sen_o_t, - src_text2=_pair_sen[3], - spacy_nlp=spacy_nlp, - train_mode=True) - src_tokens_length = len(src_tokens) - + src_tokens, src_tokens_length, label_idx = self.common_block(_pair_sen=_pair_sen, + spacy_nlp=spacy_nlp) train_dataset_list.append([src_tokens, src_tokens_length, label_idx]) print("Begin to process test data...") @@ -274,7 +279,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--train_file', type=str, required=True, help='train dataset file path') parser.add_argument('--test_file', type=str, required=True, help='test dataset file path') - parser.add_argument('--class_num', type=int, required=True, help='Dataset classe number') + parser.add_argument('--class_num', type=int, required=True, help='Dataset class number') parser.add_argument('--ngram', type=int, default=2, required=False) parser.add_argument('--max_len', type=int, required=False, help='max length sentence in dataset') parser.add_argument('--bucket', type=ast.literal_eval, default=[64, 128, 467], help='bucket sequence length.') diff --git a/model_zoo/research/cv/FaceQualityAssessment/scripts/run_eval.sh b/model_zoo/research/cv/FaceQualityAssessment/scripts/run_eval.sh index 89417082ba..bd09658778 100644 --- a/model_zoo/research/cv/FaceQualityAssessment/scripts/run_eval.sh +++ b/model_zoo/research/cv/FaceQualityAssessment/scripts/run_eval.sh @@ -61,7 +61,7 @@ export RANK_ID=0 rm -rf ${current_exec_path}/device$USE_DEVICE_ID echo 'start device '$USE_DEVICE_ID mkdir ${current_exec_path}/device$USE_DEVICE_ID -cd ${current_exec_path}/device$USE_DEVICE_ID +cd ${current_exec_path}/device$USE_DEVICE_ID || exit dev=`expr $USE_DEVICE_ID + 0` export DEVICE_ID=$dev python ${dirname_path}/${SCRIPT_NAME} \ diff --git a/model_zoo/research/cv/FaceQualityAssessment/train.py b/model_zoo/research/cv/FaceQualityAssessment/train.py index 0d6a3db436..500262b167 100644 --- a/model_zoo/research/cv/FaceQualityAssessment/train.py +++ b/model_zoo/research/cv/FaceQualityAssessment/train.py @@ -102,7 +102,7 @@ def main(args): else: param_dict_new[key] = values load_param_into_net(network, param_dict_new) - cfg.logger.info('load model %s success.' % cfg.pretrained) + cfg.logger.info('load model %s success.', cfg.pretrained) # optimizer and lr scheduler lr = warmup_step(cfg, gamma=0.9) diff --git a/model_zoo/research/cv/FaceRecognition/eval.py b/model_zoo/research/cv/FaceRecognition/eval.py index b13a7dd5d0..6e526b91af 100644 --- a/model_zoo/research/cv/FaceRecognition/eval.py +++ b/model_zoo/research/cv/FaceRecognition/eval.py @@ -328,6 +328,6 @@ if __name__ == '__main__': log_path = os.path.join(arg.ckpt_path, 'logs') arg.logger = get_logger(log_path, arg.local_rank) - arg.logger.info('Config\n\n{}\n'.format(pformat(arg))) + arg.logger.info('Config: {}'.format(pformat(arg))) main(arg) diff --git a/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_base.sh b/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_base.sh index b723b939cd..14189a42ed 100644 --- a/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_base.sh +++ b/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_base.sh @@ -42,7 +42,7 @@ mkdir ${EXECUTE_PATH}/log_standalone_graph rm -rf ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID mkdir -p ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID -cd ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID +cd ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID || exit echo "start training for rank $RANK_ID, device $USE_DEVICE_ID" env > ${EXECUTE_PATH}/log_standalone_graph/face_recognition_$USE_DEVICE_ID.log python ${EXECUTE_PATH}/../train.py \ diff --git a/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_beta.sh b/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_beta.sh index 3ec2f512d6..c0985f5c3b 100644 --- a/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_beta.sh +++ b/model_zoo/research/cv/FaceRecognition/scripts/run_standalone_train_beta.sh @@ -42,7 +42,7 @@ mkdir ${EXECUTE_PATH}/log_standalone_graph rm -rf ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID mkdir -p ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID -cd ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID +cd ${EXECUTE_PATH}/data_standalone_log_$USE_DEVICE_ID || exit echo "start training for rank $RANK_ID, device $USE_DEVICE_ID" env > ${EXECUTE_PATH}/log_standalone_graph/face_recognition_$USE_DEVICE_ID.log python ${EXECUTE_PATH}/../train.py \ diff --git a/model_zoo/research/cv/FaceRecognitionForTracking/train.py b/model_zoo/research/cv/FaceRecognitionForTracking/train.py index 273d9e01b5..5ab4e49e4c 100644 --- a/model_zoo/research/cv/FaceRecognitionForTracking/train.py +++ b/model_zoo/research/cv/FaceRecognitionForTracking/train.py @@ -90,13 +90,13 @@ def main(): cfg.logger.info('start create dataloader') de_dataset, steps_per_epoch, class_num = get_de_dataset(cfg) cfg.steps_per_epoch = steps_per_epoch - cfg.logger.info('step per epoch: %d' % cfg.steps_per_epoch) + cfg.logger.info('step per epoch: {}'.format(cfg.steps_per_epoch)) de_dataloader = de_dataset.create_tuple_iterator() - cfg.logger.info('class num original: %d' % class_num) + cfg.logger.info('class num original: {}'.format(class_num)) if class_num % 16 != 0: class_num = (class_num // 16 + 1) * 16 cfg.class_num = class_num - cfg.logger.info('change the class num to: %d' % cfg.class_num) + cfg.logger.info('change the class num to: {}'.format(cfg.class_num)) cfg.logger.info('end create dataloader') # backbone and loss @@ -119,7 +119,7 @@ def main(): else: param_dict_new[key] = values load_param_into_net(network, param_dict_new) - cfg.logger.info('load model %s success' % cfg.pretrained) + cfg.logger.info('load model %s success', cfg.pretrained) # mixed precision training network.add_flags_recursive(fp16=True) diff --git a/model_zoo/research/nlp/ternarybert/scripts/run_train.sh b/model_zoo/research/nlp/ternarybert/scripts/run_train.sh index 08f41c4e1b..7610666b0d 100644 --- a/model_zoo/research/nlp/ternarybert/scripts/run_train.sh +++ b/model_zoo/research/nlp/ternarybert/scripts/run_train.sh @@ -15,7 +15,7 @@ # ============================================================================ mkdir -p ms_log -PROJECT_DIR=$(cd "$(dirname "$0")"; pwd) +PROJECT_DIR=$(cd "$(dirname "$0")" || exit; pwd) CUR_DIR=`pwd` export GLOG_log_dir=${CUR_DIR}/ms_log export GLOG_logtostderr=0