|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208 |
- # Copyright 2021 Huawei Technologies Co., Ltd
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- # ============================================================================
-
- import numpy as np
- import mindspore
- from mindspore import nn, Tensor
- from mindspore.ops import operations as P
- from mindspore.nn.optim import ASGD
- from mindspore.nn.optim import Rprop
- np.random.seed(1024)
-
- fc1_weight = np.array([[0.72346634, 0.95608497, 0.4084163, 0.18627149,
- 0.6942514, 0.39767185, 0.24918061, 0.4548748],
- [0.7203382, 0.19086994, 0.76286614, 0.87920564,
- 0.3169892, 0.9462494, 0.62827677, 0.27504718],
- [0.3544535, 0.2524781, 0.5370583, 0.8313121,
- 0.6670143, 0.0488653, 0.62225235, 0.7546456],
- [0.17985944, 0.05106374, 0.31064633, 0.4863033,
- 0.848814, 0.5523157, 0.20295663, 0.7213356]]).astype("float32")
-
- fc1_bias = np.array([0.79708564, 0.13728078, 0.66322654, 0.88128525]).astype("float32")
-
- fc2_weight = np.array([[0.8473515, 0.50923985, 0.42287776, 0.29769543]]).astype("float32")
-
- fc2_bias = np.array([0.09996348]).astype("float32")
-
-
- def make_fake_data():
- """
- make fake data
- """
- data, label = [], []
- for i in range(20):
- data.append(mindspore.Tensor(np.array(np.ones((2, 8)) * i, dtype=np.float32)))
- label.append(mindspore.Tensor(np.array(np.ones((2, 1)) * (i + 1), dtype=np.float32)))
- return data, label
-
-
- class NetWithLoss(nn.Cell):
- """
- build net with loss
- """
- def __init__(self, network):
- super(NetWithLoss, self).__init__()
- self.network = network
- self.loss = nn.MSELoss(reduction='sum')
-
- def construct(self, x, label):
- out = self.network(x)
- loss = self.loss(out, label)
- return loss
-
-
- class FakeNet(nn.Cell):
- """
- build fake net
- """
- def __init__(self):
- super(FakeNet, self).__init__()
- self.fc1 = nn.Dense(in_channels=8, out_channels=4, weight_init=Tensor(fc1_weight), bias_init=Tensor(fc1_bias))
- self.fc2 = nn.Dense(in_channels=4, out_channels=1, weight_init=Tensor(fc2_weight), bias_init=Tensor(fc2_bias))
- self.relu = nn.ReLU()
- self.reducemean = P.ReduceMean()
-
- def construct(self, x):
- x = self.relu(self.fc1(x))
- x = self.fc2(x)
- return x
-
- def _initialize_weights(self):
- """
- parameter initialization
- """
- self.init_parameters_data()
- for name, m in self.cells_and_names():
- if name == 'fc1':
- m.weight.set_data(Tensor(fc1_weight))
- m.bias.set_data(Tensor(fc1_bias))
- elif name == 'fc2':
- m.weight.set_data(Tensor(fc2_weight))
- m.bias.set_data(Tensor(fc2_bias))
-
-
- def build_network(opt_config, is_group=False):
- """
- Construct training
- """
- losses = []
- net = FakeNet()
-
- networkwithloss = NetWithLoss(net)
- networkwithloss.set_train()
-
- if is_group:
- fc1_params = list(filter(lambda x: 'fc1' in x.name, networkwithloss.trainable_params()))
- fc2_params = list(filter(lambda x: 'fc1' not in x.name, networkwithloss.trainable_params()))
- if opt_config['name'] == 'ASGD':
- params = [{'params': fc1_params, 'weight_decay': 0.01, 'lr': 0.001}, {'params': fc2_params, 'lr': 0.1}]
- else:
- params = [{'params': fc1_params, 'lr': 0.001}, {'params': fc2_params, 'lr': 0.1}]
- else:
- params = networkwithloss.trainable_params()
-
- if opt_config['name'] == 'ASGD':
- net_opt = ASGD(params, learning_rate=opt_config['lr'], lambd=opt_config['lambd'], alpha=opt_config['alpha'],
- t0=opt_config['t0'], weight_decay=opt_config['weight_decay'])
-
- elif opt_config['name'] == 'Rprop':
- net_opt = Rprop(params, learning_rate=opt_config['lr'], etas=opt_config['etas'],
- step_sizes=opt_config['step_sizes'], weight_decay=0.0)
-
- trainonestepcell = mindspore.nn.TrainOneStepCell(networkwithloss, net_opt)
- data, label = make_fake_data()
- for i in range(20):
- loss = trainonestepcell(data[i], label[i])
- losses.append(loss.asnumpy())
- if opt_config['name'] == 'ASGD':
- return np.array(losses), net_opt
- return np.array(losses)
-
-
- loss_default_asgd = np.array([3.01246792e-01, 1.20041794e+02, 1.38681079e+03, 2.01250820e+01,
- 3.27283554e+01, 4.76963005e+01, 6.47094269e+01, 8.34786530e+01,
- 1.03742706e+02, 1.25265739e+02, 1.47835190e+02, 1.71259613e+02,
- 1.95367035e+02, 2.20003204e+02, 2.45029831e+02, 2.70323456e+02,
- 2.95774048e+02, 3.21283752e+02, 3.46765594e+02, 3.72143097e+02], dtype=np.float32)
-
- loss_not_default_asgd = np.array([3.01246792e-01, 1.26019104e+02, 1.90600449e+02, 9.70605755e+00,
- 2.98419113e+01, 3.68430023e+02, 1.06318066e+04, 1.35017746e+02,
- 1.68673813e+02, 2.05914215e+02, 2.46694992e+02, 2.90972443e+02,
- 3.38703430e+02, 3.89845123e+02, 4.44355103e+02, 5.02191406e+02,
- 5.63312500e+02, 6.27676941e+02, 6.95244202e+02, 7.65973816e+02], dtype=np.float32)
-
- loss_group_asgd = np.array([3.01246792e-01, 7.26708527e+01, 2.84905312e+05, 4.17499258e+04,
- 1.46797949e+04, 5.07966602e+03, 1.70935132e+03, 5.47094910e+02,
- 1.59216995e+02, 3.78818207e+01, 5.18196869e+00, 2.62275129e-03,
- 2.09768796e+00, 5.23108435e+00, 7.78943682e+00, 9.57108879e+00,
- 1.07310610e+01, 1.14618425e+01, 1.19147835e+01, 1.21936722e+01], dtype=np.float32)
-
-
- loss_default_rprop = np.array([3.01246792e-01, 1.19871742e+02, 4.13467163e+02, 8.09146179e+02,
- 1.22364807e+03, 1.56787573e+03, 1.75733594e+03, 1.72866272e+03,
- 1.46183936e+03, 1.00406335e+03, 4.84076874e+02, 9.49734650e+01,
- 2.00592804e+01, 1.87920704e+01, 1.53733969e+01, 1.85836582e+01,
- 5.21527790e-02, 2.01522671e-02, 7.19913816e+00, 8.52459526e+00], dtype=np.float32)
-
- loss_not_default_rprop = np.array([3.0124679e-01, 1.2600269e+02, 4.7351608e+02, 1.0220379e+03,
- 1.7181555e+03, 2.4367019e+03, 2.9170872e+03, 2.7243464e+03,
- 1.4999669e+03, 7.5820435e+01, 1.0590715e+03, 5.4336096e+02,
- 7.0162407e+01, 8.2754419e+02, 9.6329260e+02, 3.4475109e+01,
- 5.3843134e+02, 6.0064526e+02, 1.1046149e+02, 3.5530117e+03], dtype=np.float32)
-
- loss_group_rprop = np.array([3.0124679e-01, 7.1360558e+01, 4.8910957e+01, 2.1730331e+02,
- 3.0747052e+02, 5.2734237e+00, 5.6865869e+00, 1.7116127e+02,
- 2.0539343e+02, 2.2993685e+01, 2.6194101e+02, 2.8772815e+02,
- 2.4236647e+01, 3.9299741e+02, 3.5600668e+02, 1.4759110e+01,
- 7.2244568e+02, 8.1952783e+02, 9.8913864e+01, 1.1141744e+03], dtype=np.float32)
-
-
- default_fc1_weight_asgd = np.array([[-0.9451941, -0.71258026, -1.2602371, -1.4823773,
- -0.974408, -1.2709816, -1.4194703, -1.2137808],
- [-1.5341775, -2.0636342, -1.4916497, -1.3753126,
- -1.9375193, -1.308271, -1.6262367, -1.9794592],
- [-1.9886293, -2.0906024, -1.8060291, -1.5117803,
- -1.6760755, -2.2942104, -1.7208353, -1.5884445],
- [-2.071215, -2.2000103, -1.9404325, -1.7647781,
- -1.4022746, -1.6987679, -2.0481179, -1.5297506]], dtype=np.float32)
- default_fc1_bias_asgd = np.array([-0.17978168, -1.0764512, -0.578816, -0.2928958], dtype=np.float32)
- default_fc2_weight_asgd = np.array([[4.097412, 6.2694297, 5.9203916, 5.3845487]], dtype=np.float32)
- default_fc2_bias_asgd = np.array([6.904814], dtype=np.float32)
-
-
- no_default_fc1_weight_asgd = np.array([[-1.3406217, -1.1080127, -1.655658, -1.8777936,
- -1.3698348, -1.6664025, -1.8148884, -1.6092018],
- [-1.1475986, -1.6770473, -1.1050745, -0.98873824,
- -1.5509329, -0.9216978, -1.2396574, -1.5928726],
- [-1.2329121, -1.334883, -1.050313, -0.756071,
- -0.92036265, -1.5384867, -0.96512324, -0.8327349],
- [-1.0685704, -1.1973612, -0.9377885, -0.7621386,
- -0.39964262, -0.69612867, -1.0454736, -0.52711576]], dtype=np.float32)
- no_default_fc1_bias_asgd = np.array([0.41264832, -0.19961096, 0.37743938, 0.65807366], dtype=np.float32)
- no_default_fc2_weight_asgd = np.array([[-5.660916, -5.9415145, -5.1402636, -4.199707]], dtype=np.float32)
- no_default_fc2_bias_asgd = np.array([0.5082278], dtype=np.float32)
-
-
- no_default_group_fc1_weight_asgd = np.array([[-32.526627, -32.29401, -32.8416, -33.06367, -32.55584,
- -32.852345, -33.000767, -32.795143],
- [-33.164936, -33.69432, -33.12241, -33.006073, -33.568207,
- -32.9391, -33.256996, -33.61015],
- [-33.118973, -33.220943, -32.936436, -32.642193, -32.806488,
- -33.424484, -32.85125, -32.718857],
- [-30.155754, -30.284513, -30.025005, -29.849358, -29.486917,
- -29.783375, -30.132658, -29.614393]], dtype=np.float32)
- no_default_group_fc1_bias_asgd = np.array([-15.838092, -16.811989, -16.078112, -14.289094], dtype=np.float32)
- no_default_group_fc2_weight_asgd = np.array([[1288.7146, 1399.3041, 1292.8445, 1121.4629]], dtype=np.float32)
- no_default_group_fc2_bias_asgd = np.array([18.513494], dtype=np.float32)
|
