zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 438e261bbc
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '438e261bbc'
${ noResults }
mindspore2022/tests/st/optimizer/test_dynamic_weight_decay_g...

185 lines
8.9 kB
Raw Blame History 

  1. # Copyright 2022 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ============================================================================

  15. import mindspore.context as context

  16. import mindspore.nn as nn

  17. from .weight_decay_utils import dynamic_weight_decay_cmp, WeightDecaySchdule, Net

  18. 

  19. 

  20. def test_momentum_dynamic_weight_decay_pynative():

  21.     """

  22.     Feature: Dynamic weight decay

  23.     Description: Test dynamic weight decay for Momentum

  24.     Expectation: The value of decay changes according to preset weight decay schedule

  25.     """

  26.     context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")

  27.     net1, net2 = Net(), Net()

  28.     weight_decay_schedule = WeightDecaySchdule()

  29.     optimizer1 = nn.Momentum(net1.trainable_params(), momentum=0.001, learning_rate=0.001, weight_decay=0.001)

  30.     optimizer2 = nn.Momentum(net2.trainable_params(), momentum=0.001, learning_rate=0.001,

  31.                              weight_decay=weight_decay_schedule)

  32.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  33. 

  34. 

  35. def test_momentum_dynamic_weight_decay_graph():

  36.     """

  37.     Feature: Dynamic weight decay

  38.     Description: Test dynamic weight decay for Momentum

  39.     Expectation: The value of decay changes according to preset weight decay schedule

  40.     """

  41.     context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

  42.     net1, net2 = Net(), Net()

  43.     weight_decay_schedule = WeightDecaySchdule()

  44.     optimizer1 = nn.Momentum(net1.trainable_params(), momentum=0.001, learning_rate=0.001, weight_decay=0.001)

  45.     optimizer2 = nn.Momentum(net2.trainable_params(), momentum=0.001, learning_rate=0.001,

  46.                              weight_decay=weight_decay_schedule)

  47.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  48. 

  49. 

  50. def test_momentum_dynamic_weight_decay_graph_group():

  51.     """

  52.     Feature: Dynamic weight decay

  53.     Description: Test dynamic weight decay for Momentum

  54.     Expectation: The value of decay changes according to preset weight decay schedule

  55.     """

  56.     context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

  57.     weight_decay_schedule = WeightDecaySchdule()

  58.     net1, net2 = Net(), Net()

  59. 

  60.     net1_fc1_params = list(filter(lambda x: 'fc1' in x.name, net1.trainable_params()))

  61.     net1_fc2_params = list(filter(lambda x: 'fc1' not in x.name, net1.trainable_params()))

  62. 

  63.     net2_fc1_params = list(filter(lambda x: 'fc1' in x.name, net2.trainable_params()))

  64.     net2_fc2_params = list(filter(lambda x: 'fc1' not in x.name, net2.trainable_params()))

  65. 

  66.     params1 = [{'params': net1_fc1_params, 'weight_decay': 0.01, 'lr': 0.01},

  67.                {'params': net1_fc2_params, 'weight_decay': 0.001, 'lr': 0.001}]

  68. 

  69.     params2 = [{'params': net2_fc1_params, 'weight_decay': 0.01, 'lr': 0.01},

  70.                {'params': net2_fc2_params, 'weight_decay': weight_decay_schedule, 'lr': 0.001}]

  71. 

  72.     optimizer1 = nn.Momentum(params1, momentum=0.001, learning_rate=0.001, weight_decay=0.001)

  73.     optimizer2 = nn.Momentum(params2, momentum=0.001, learning_rate=0.001, weight_decay=0.001)

  74.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  75. 

  76. 

  77. def test_adamweightdecay_dynamic_weight_decay_pynative():

  78.     """

  79.     Feature: Dynamic weight decay

  80.     Description: Test dynamic weight decay for AdamWeightDecay

  81.     Expectation: The value of decay changes according to preset weight decay schedule

  82.     """

  83.     context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")

  84.     net1, net2 = Net(), Net()

  85.     weight_decay_schedule = WeightDecaySchdule()

  86.     optimizer1 = nn.AdamWeightDecay(net1.trainable_params(), learning_rate=0.001, weight_decay=0.001)

  87.     optimizer2 = nn.AdamWeightDecay(net2.trainable_params(), learning_rate=0.001, weight_decay=weight_decay_schedule)

  88.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  89. 

  90. 

  91. def test_adamweightdecay_dynamic_weight_decay_graph():

  92.     """

  93.     Feature: Dynamic weight decay

  94.     Description: Test dynamic weight decay for AdamWeightDecay

  95.     Expectation: The value of decay changes according to preset weight decay schedule

  96.     """

  97.     context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

  98.     net1, net2 = Net(), Net()

  99.     weight_decay_schedule = WeightDecaySchdule()

  100.     optimizer1 = nn.AdamWeightDecay(net1.trainable_params(), learning_rate=0.001, weight_decay=0.001)

  101.     optimizer2 = nn.AdamWeightDecay(net2.trainable_params(), learning_rate=0.001, weight_decay=weight_decay_schedule)

  102.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  103. 

  104. 

  105. def test_adamweightdecay_dynamic_weight_decay_graph_group():

  106.     """

  107.     Feature: Dynamic weight decay

  108.     Description: Test dynamic weight decay for Momentum

  109.     Expectation: The value of decay changes according to preset weight decay schedule

  110.     """

  111.     context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

  112.     weight_decay_schedule = WeightDecaySchdule()

  113.     net1, net2 = Net(), Net()

  114. 

  115.     net1_fc1_params = list(filter(lambda x: 'fc1' in x.name, net1.trainable_params()))

  116.     net1_fc2_params = list(filter(lambda x: 'fc1' not in x.name, net1.trainable_params()))

  117. 

  118.     net2_fc1_params = list(filter(lambda x: 'fc1' in x.name, net2.trainable_params()))

  119.     net2_fc2_params = list(filter(lambda x: 'fc1' not in x.name, net2.trainable_params()))

  120. 

  121.     params1 = [{'params': net1_fc1_params, 'weight_decay': 0.01, 'lr': 0.01},

  122.                {'params': net1_fc2_params, 'weight_decay': 0.001, 'lr': 0.001}]

  123. 

  124.     params2 = [{'params': net2_fc1_params, 'weight_decay': 0.01, 'lr': 0.01},

  125.                {'params': net2_fc2_params, 'weight_decay': weight_decay_schedule, 'lr': 0.001}]

  126. 

  127.     optimizer1 = nn.AdamWeightDecay(params1, learning_rate=0.001, weight_decay=0.001)

  128.     optimizer2 = nn.AdamWeightDecay(params2, learning_rate=0.001, weight_decay=0.001)

  129.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  130. 

  131. 

  132. def test_lamb_dynamic_weight_decay_pynative():

  133.     """

  134.     Feature: Dynamic weight decay

  135.     Description: Test dynamic weight decay for Lamb

  136.     Expectation: The value of decay changes according to preset weight decay schedule

  137.     """

  138.     context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")

  139.     net1, net2 = Net(), Net()

  140.     weight_decay_schedule = WeightDecaySchdule()

  141.     optimizer1 = nn.Lamb(net1.trainable_params(), learning_rate=0.001, weight_decay=0.001)

  142.     optimizer2 = nn.Lamb(net2.trainable_params(), learning_rate=0.001, weight_decay=weight_decay_schedule)

  143.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  144. 

  145. 

  146. def test_lamb_dynamic_weight_decay_graph():

  147.     """

  148.     Feature: Dynamic weight decay

  149.     Description: Test dynamic weight decay for Lamb

  150.     Expectation: The value of decay changes according to preset weight decay schedule

  151.     """

  152.     context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

  153.     net1, net2 = Net(), Net()

  154.     weight_decay_schedule = WeightDecaySchdule()

  155.     optimizer1 = nn.Lamb(net1.trainable_params(), learning_rate=0.001, weight_decay=0.001)

  156.     optimizer2 = nn.Lamb(net2.trainable_params(), learning_rate=0.001, weight_decay=weight_decay_schedule)

  157.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)

  158. 

  159. 

  160. def test_lamb_dynamic_weight_decay_graph_group():

  161.     """

  162.     Feature: Dynamic weight decay

  163.     Description: Test dynamic weight decay for Momentum

  164.     Expectation: The value of decay changes according to preset weight decay schedule

  165.     """

  166.     context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

  167.     weight_decay_schedule = WeightDecaySchdule()

  168.     net1, net2 = Net(), Net()

  169. 

  170.     net1_fc1_params = list(filter(lambda x: 'fc1' in x.name, net1.trainable_params()))

  171.     net1_fc2_params = list(filter(lambda x: 'fc1' not in x.name, net1.trainable_params()))

  172. 

  173.     net2_fc1_params = list(filter(lambda x: 'fc1' in x.name, net2.trainable_params()))

  174.     net2_fc2_params = list(filter(lambda x: 'fc1' not in x.name, net2.trainable_params()))

  175. 

  176.     params1 = [{'params': net1_fc1_params, 'weight_decay': 0.01, 'lr': 0.01},

  177.                {'params': net1_fc2_params, 'weight_decay': 0.001, 'lr': 0.001}]

  178. 

  179.     params2 = [{'params': net2_fc1_params, 'weight_decay': 0.01, 'lr': 0.01},

  180.                {'params': net2_fc2_params, 'weight_decay': weight_decay_schedule, 'lr': 0.001}]

  181. 

  182.     optimizer1 = nn.Lamb(params1, learning_rate=0.001, weight_decay=0.001)

  183.     optimizer2 = nn.Lamb(params2, learning_rate=0.001, weight_decay=0.001)

  184.     dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2)