From 2f05a0a441f8c74d7236fdef58f1dcf006de292c Mon Sep 17 00:00:00 2001 From: chenhaozhe Date: Thu, 11 Mar 2021 15:24:24 +0800 Subject: [PATCH] Increase num_parallel in data preprocess of mobilenetv1 --- model_zoo/official/cv/mobilenetv1/src/dataset.py | 16 ++++++++-------- model_zoo/official/cv/mobilenetv1/train.py | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/model_zoo/official/cv/mobilenetv1/src/dataset.py b/model_zoo/official/cv/mobilenetv1/src/dataset.py index 305a698d9e..be81f3253f 100755 --- a/model_zoo/official/cv/mobilenetv1/src/dataset.py +++ b/model_zoo/official/cv/mobilenetv1/src/dataset.py @@ -44,9 +44,9 @@ def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target= device_num = get_group_size() if device_num == 1: - data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) + data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=12, shuffle=True) else: - data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, + data_set = ds.Cifar10Dataset(dataset_path, num_parallel_workers=12, shuffle=True, num_shards=device_num, shard_id=rank_id) # define map operations @@ -66,8 +66,8 @@ def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target= type_cast_op = C2.TypeCast(mstype.int32) - data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) - data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) + data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=12) + data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=12) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) @@ -99,9 +99,9 @@ def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target= device_num = get_group_size() if device_num == 1: - data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) + data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True) else: - data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + data_set = ds.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True, num_shards=device_num, shard_id=rank_id) image_size = 224 @@ -127,8 +127,8 @@ def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target= type_cast_op = C2.TypeCast(mstype.int32) - data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=8) - data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8) + data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=12) + data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=12) # apply batch operations data_set = data_set.batch(batch_size, drop_remainder=True) diff --git a/model_zoo/official/cv/mobilenetv1/train.py b/model_zoo/official/cv/mobilenetv1/train.py index 41f9377c82..ba2f378912 100755 --- a/model_zoo/official/cv/mobilenetv1/train.py +++ b/model_zoo/official/cv/mobilenetv1/train.py @@ -68,6 +68,7 @@ if __name__ == '__main__': context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True) init() + context.set_auto_parallel_context(all_reduce_fusion_config=[75]) # GPU target else: init()