zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 03b32e4eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '03b32e4eab'
${ noResults }
mindspore2022/tests/st/optimizer/optimizer_utils.py

209 lines
11 kB
Raw Blame History 

  1. # Copyright 2021 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ============================================================================

  15. 

  16. import numpy as np

  17. import mindspore

  18. from mindspore import nn, Tensor

  19. from mindspore.ops import operations as P

  20. from mindspore.nn.optim import ASGD

  21. from mindspore.nn.optim import Rprop

  22. np.random.seed(1024)

  23. 

  24. fc1_weight = np.array([[0.72346634, 0.95608497, 0.4084163, 0.18627149,

  25.                         0.6942514, 0.39767185, 0.24918061, 0.4548748],

  26.                        [0.7203382, 0.19086994, 0.76286614, 0.87920564,

  27.                         0.3169892, 0.9462494, 0.62827677, 0.27504718],

  28.                        [0.3544535, 0.2524781, 0.5370583, 0.8313121,

  29.                         0.6670143, 0.0488653, 0.62225235, 0.7546456],

  30.                        [0.17985944, 0.05106374, 0.31064633, 0.4863033,

  31.                         0.848814, 0.5523157, 0.20295663, 0.7213356]]).astype("float32")

  32. 

  33. fc1_bias = np.array([0.79708564, 0.13728078, 0.66322654, 0.88128525]).astype("float32")

  34. 

  35. fc2_weight = np.array([[0.8473515, 0.50923985, 0.42287776, 0.29769543]]).astype("float32")

  36. 

  37. fc2_bias = np.array([0.09996348]).astype("float32")

  38. 

  39. 

  40. def make_fake_data():

  41.     """

  42.     make fake data

  43.     """

  44.     data, label = [], []

  45.     for i in range(20):

  46.         data.append(mindspore.Tensor(np.array(np.ones((2, 8)) * i, dtype=np.float32)))

  47.         label.append(mindspore.Tensor(np.array(np.ones((2, 1)) * (i + 1), dtype=np.float32)))

  48.     return data, label

  49. 

  50. 

  51. class NetWithLoss(nn.Cell):

  52.     """

  53.     build net with loss

  54.     """

  55.     def __init__(self, network):

  56.         super(NetWithLoss, self).__init__()

  57.         self.network = network

  58.         self.loss = nn.MSELoss(reduction='sum')

  59. 

  60.     def construct(self, x, label):

  61.         out = self.network(x)

  62.         loss = self.loss(out, label)

  63.         return loss

  64. 

  65. 

  66. class FakeNet(nn.Cell):

  67.     """

  68.     build fake net

  69.     """

  70.     def __init__(self):

  71.         super(FakeNet, self).__init__()

  72.         self.fc1 = nn.Dense(in_channels=8, out_channels=4, weight_init=Tensor(fc1_weight), bias_init=Tensor(fc1_bias))

  73.         self.fc2 = nn.Dense(in_channels=4, out_channels=1, weight_init=Tensor(fc2_weight), bias_init=Tensor(fc2_bias))

  74.         self.relu = nn.ReLU()

  75.         self.reducemean = P.ReduceMean()

  76. 

  77.     def construct(self, x):

  78.         x = self.relu(self.fc1(x))

  79.         x = self.fc2(x)

  80.         return x

  81. 

  82.     def _initialize_weights(self):

  83.         """

  84.         parameter initialization

  85.         """

  86.         self.init_parameters_data()

  87.         for name, m in self.cells_and_names():

  88.             if name == 'fc1':

  89.                 m.weight.set_data(Tensor(fc1_weight))

  90.                 m.bias.set_data(Tensor(fc1_bias))

  91.             elif name == 'fc2':

  92.                 m.weight.set_data(Tensor(fc2_weight))

  93.                 m.bias.set_data(Tensor(fc2_bias))

  94. 

  95. 

  96. def build_network(opt_config, is_group=False):

  97.     """

  98.     Construct training

  99.     """

  100.     losses = []

  101.     net = FakeNet()

  102. 

  103.     networkwithloss = NetWithLoss(net)

  104.     networkwithloss.set_train()

  105. 

  106.     if is_group:

  107.         fc1_params = list(filter(lambda x: 'fc1' in x.name, networkwithloss.trainable_params()))

  108.         fc2_params = list(filter(lambda x: 'fc1' not in x.name, networkwithloss.trainable_params()))

  109.         if opt_config['name'] == 'ASGD':

  110.             params = [{'params': fc1_params, 'weight_decay': 0.01, 'lr': 0.001}, {'params': fc2_params, 'lr': 0.1}]

  111.         else:

  112.             params = [{'params': fc1_params, 'lr': 0.001}, {'params': fc2_params, 'lr': 0.1}]

  113.     else:

  114.         params = networkwithloss.trainable_params()

  115. 

  116.     if opt_config['name'] == 'ASGD':

  117.         net_opt = ASGD(params, learning_rate=opt_config['lr'], lambd=opt_config['lambd'], alpha=opt_config['alpha'],

  118.                        t0=opt_config['t0'], weight_decay=opt_config['weight_decay'])

  119. 

  120.     elif opt_config['name'] == 'Rprop':

  121.         net_opt = Rprop(params, learning_rate=opt_config['lr'], etas=opt_config['etas'],

  122.                         step_sizes=opt_config['step_sizes'], weight_decay=0.0)

  123. 

  124.     trainonestepcell = mindspore.nn.TrainOneStepCell(networkwithloss, net_opt)

  125.     data, label = make_fake_data()

  126.     for i in range(20):

  127.         loss = trainonestepcell(data[i], label[i])

  128.         losses.append(loss.asnumpy())

  129.     if opt_config['name'] == 'ASGD':

  130.         return np.array(losses), net_opt

  131.     return np.array(losses)

  132. 

  133. 

  134. loss_default_asgd = np.array([3.01246792e-01, 1.20041794e+02, 1.38681079e+03, 2.01250820e+01,

  135.                               3.27283554e+01, 4.76963005e+01, 6.47094269e+01, 8.34786530e+01,

  136.                               1.03742706e+02, 1.25265739e+02, 1.47835190e+02, 1.71259613e+02,

  137.                               1.95367035e+02, 2.20003204e+02, 2.45029831e+02, 2.70323456e+02,

  138.                               2.95774048e+02, 3.21283752e+02, 3.46765594e+02, 3.72143097e+02], dtype=np.float32)

  139. 

  140. loss_not_default_asgd = np.array([3.01246792e-01, 1.26019104e+02, 1.90600449e+02, 9.70605755e+00,

  141.                                   2.98419113e+01, 3.68430023e+02, 1.06318066e+04, 1.35017746e+02,

  142.                                   1.68673813e+02, 2.05914215e+02, 2.46694992e+02, 2.90972443e+02,

  143.                                   3.38703430e+02, 3.89845123e+02, 4.44355103e+02, 5.02191406e+02,

  144.                                   5.63312500e+02, 6.27676941e+02, 6.95244202e+02, 7.65973816e+02], dtype=np.float32)

  145. 

  146. loss_group_asgd = np.array([3.01246792e-01, 7.26708527e+01, 2.84905312e+05, 4.17499258e+04,

  147.                             1.46797949e+04, 5.07966602e+03, 1.70935132e+03, 5.47094910e+02,

  148.                             1.59216995e+02, 3.78818207e+01, 5.18196869e+00, 2.62275129e-03,

  149.                             2.09768796e+00, 5.23108435e+00, 7.78943682e+00, 9.57108879e+00,

  150.                             1.07310610e+01, 1.14618425e+01, 1.19147835e+01, 1.21936722e+01], dtype=np.float32)

  151. 

  152. 

  153. loss_default_rprop = np.array([3.01246792e-01, 1.19871742e+02, 4.13467163e+02, 8.09146179e+02,

  154.                                1.22364807e+03, 1.56787573e+03, 1.75733594e+03, 1.72866272e+03,

  155.                                1.46183936e+03, 1.00406335e+03, 4.84076874e+02, 9.49734650e+01,

  156.                                2.00592804e+01, 1.87920704e+01, 1.53733969e+01, 1.85836582e+01,

  157.                                5.21527790e-02, 2.01522671e-02, 7.19913816e+00, 8.52459526e+00], dtype=np.float32)

  158. 

  159. loss_not_default_rprop = np.array([3.0124679e-01, 1.2600269e+02, 4.7351608e+02, 1.0220379e+03,

  160.                                    1.7181555e+03, 2.4367019e+03, 2.9170872e+03, 2.7243464e+03,

  161.                                    1.4999669e+03, 7.5820435e+01, 1.0590715e+03, 5.4336096e+02,

  162.                                    7.0162407e+01, 8.2754419e+02, 9.6329260e+02, 3.4475109e+01,

  163.                                    5.3843134e+02, 6.0064526e+02, 1.1046149e+02, 3.5530117e+03], dtype=np.float32)

  164. 

  165. loss_group_rprop = np.array([3.0124679e-01, 7.1360558e+01, 4.8910957e+01, 2.1730331e+02,

  166.                              3.0747052e+02, 5.2734237e+00, 5.6865869e+00, 1.7116127e+02,

  167.                              2.0539343e+02, 2.2993685e+01, 2.6194101e+02, 2.8772815e+02,

  168.                              2.4236647e+01, 3.9299741e+02, 3.5600668e+02, 1.4759110e+01,

  169.                              7.2244568e+02, 8.1952783e+02, 9.8913864e+01, 1.1141744e+03], dtype=np.float32)

  170. 

  171. 

  172. default_fc1_weight_asgd = np.array([[-0.9451941, -0.71258026, -1.2602371, -1.4823773,

  173.                                      -0.974408, -1.2709816, -1.4194703, -1.2137808],

  174.                                     [-1.5341775, -2.0636342, -1.4916497, -1.3753126,

  175.                                      -1.9375193, -1.308271, -1.6262367, -1.9794592],

  176.                                     [-1.9886293, -2.0906024, -1.8060291, -1.5117803,

  177.                                      -1.6760755, -2.2942104, -1.7208353, -1.5884445],

  178.                                     [-2.071215, -2.2000103, -1.9404325, -1.7647781,

  179.                                      -1.4022746, -1.6987679, -2.0481179, -1.5297506]], dtype=np.float32)

  180. default_fc1_bias_asgd = np.array([-0.17978168, -1.0764512, -0.578816, -0.2928958], dtype=np.float32)

  181. default_fc2_weight_asgd = np.array([[4.097412, 6.2694297, 5.9203916, 5.3845487]], dtype=np.float32)

  182. default_fc2_bias_asgd = np.array([6.904814], dtype=np.float32)

  183. 

  184. 

  185. no_default_fc1_weight_asgd = np.array([[-1.3406217, -1.1080127, -1.655658, -1.8777936,

  186.                                         -1.3698348, -1.6664025, -1.8148884, -1.6092018],

  187.                                        [-1.1475986, -1.6770473, -1.1050745, -0.98873824,

  188.                                         -1.5509329, -0.9216978, -1.2396574, -1.5928726],

  189.                                        [-1.2329121, -1.334883, -1.050313, -0.756071,

  190.                                         -0.92036265, -1.5384867, -0.96512324, -0.8327349],

  191.                                        [-1.0685704, -1.1973612, -0.9377885, -0.7621386,

  192.                                         -0.39964262, -0.69612867, -1.0454736, -0.52711576]], dtype=np.float32)

  193. no_default_fc1_bias_asgd = np.array([0.41264832, -0.19961096, 0.37743938, 0.65807366], dtype=np.float32)

  194. no_default_fc2_weight_asgd = np.array([[-5.660916, -5.9415145, -5.1402636, -4.199707]], dtype=np.float32)

  195. no_default_fc2_bias_asgd = np.array([0.5082278], dtype=np.float32)

  196. 

  197. 

  198. no_default_group_fc1_weight_asgd = np.array([[-32.526627, -32.29401, -32.8416, -33.06367, -32.55584,

  199.                                               -32.852345, -33.000767, -32.795143],

  200.                                              [-33.164936, -33.69432, -33.12241, -33.006073, -33.568207,

  201.                                               -32.9391, -33.256996, -33.61015],

  202.                                              [-33.118973, -33.220943, -32.936436, -32.642193, -32.806488,

  203.                                               -33.424484, -32.85125, -32.718857],

  204.                                              [-30.155754, -30.284513, -30.025005, -29.849358, -29.486917,

  205.                                               -29.783375, -30.132658, -29.614393]], dtype=np.float32)

  206. no_default_group_fc1_bias_asgd = np.array([-15.838092, -16.811989, -16.078112, -14.289094], dtype=np.float32)

  207. no_default_group_fc2_weight_asgd = np.array([[1288.7146, 1399.3041, 1292.8445, 1121.4629]], dtype=np.float32)

  208. no_default_group_fc2_bias_asgd = np.array([18.513494], dtype=np.float32)