Browse Source

!13372 fix grad accumulation bug for no used parameters

From: @yangzhenzhang
Reviewed-by: @kisnwang,@stsuteng
Signed-off-by: @stsuteng
tags/v1.2.0-rc1
mindspore-ci-bot Gitee 4 years ago
parent
commit
e091e8cad6
1 changed files with 9 additions and 0 deletions
  1. +9
    -0
      mindspore/ccsrc/frontend/parallel/step_parallel.cc

+ 9
- 0
mindspore/ccsrc/frontend/parallel/step_parallel.cc View File

@@ -3325,6 +3325,15 @@ static void HandleNoUsedParameter(const FuncGraphPtr &root) {
if (full_batch) { if (full_batch) {
return; return;
} }

// in grad accumulation mode, if use dynamic lr, it has some parameters in optimizer which no used for first graph,
// but used for second graph(such as global_step), so can not change their shapes
int64_t grad_accumulation_step = ParallelContext::GetInstance()->grad_accumulation_step();
if (grad_accumulation_step > 1) {
MS_LOG(INFO) << "In grad accumulation mode, do not handle no used parameters";
return;
}

auto dev_num = g_device_manager->stage_device_num(); auto dev_num = g_device_manager->stage_device_num();
auto parameters = root->parameters(); auto parameters = root->parameters();
for (auto &parameter : parameters) { for (auto &parameter : parameters) {


Loading…
Cancel
Save