!26786 Add initializer and optimizer ut

Merge pull request !26786 from wanyiming/add_init_opt_ut

tests/st/initializer/test_initializer.py

@@ -24,7 +24,8 @@ def test_sparse():
     Description: Initialize a 2 dimension sparse matrix to fill the input tensor.
     Expectation: The Tensor is initialized with a 2 dimension sparse matrix.
     """
-    initializer(Sparse(sparsity=0.1, sigma=0.01), [5, 8], mindspore.float32)
+    tensor1 = initializer(Sparse(sparsity=0.1, sigma=0.01), [5, 8], mindspore.float32)
+    tensor1.init_data()
 
 
 def test_orthogonal():
@@ -33,8 +34,10 @@ def test_orthogonal():
     Description: Initialize a (semi) orthogonal matrix to fill the input tensor.
     Expectation: The Tensor is initialized with values from orthogonal matrix.
     """
-    initializer(Orthogonal(gain=2.), [2, 3, 4], mindspore.float32)
-    initializer('orthogonal', [2, 3, 4], mindspore.float32)
+    tensor1 = initializer(Orthogonal(gain=2.), [2, 3, 4], mindspore.float32)
+    tensor2 = initializer('orthogonal', [2, 3, 4], mindspore.float32)
+    tensor1.init_data()
+    tensor2.init_data()
 
 
 def test_variancescaling():
@@ -43,13 +46,17 @@ def test_variancescaling():
     Description: Randomly initialize an array with scaling to fill the input tensor.
     Expectation: The Tensor is initialized successfully.
     """
-    initializer('varianceScaling', [2, 3], mindspore.float32)
-    initializer(VarianceScaling(scale=1.0, mode='fan_out', distribution='untruncated_normal'), [2, 3],
-                mindspore.float32)
-    initializer(VarianceScaling(scale=2.0, mode='fan_in', distribution='truncated_normal'), [2, 3],
-                mindspore.float32)
-    initializer(VarianceScaling(scale=3.0, mode='fan_avg', distribution='uniform'), [2, 3],
-                mindspore.float32)
+    tensor1 = initializer('varianceScaling', [2, 3], mindspore.float32)
+    tensor2 = initializer(VarianceScaling(scale=1.0, mode='fan_out', distribution='untruncated_normal'), [2, 3],
+                          mindspore.float32)
+    tensor3 = initializer(VarianceScaling(scale=2.0, mode='fan_in', distribution='truncated_normal'), [2, 3],
+                          mindspore.float32)
+    tensor4 = initializer(VarianceScaling(scale=3.0, mode='fan_avg', distribution='uniform'), [2, 3],
+                          mindspore.float32)
+    tensor1.init_data()
+    tensor2.init_data()
+    tensor3.init_data()
+    tensor4.init_data()
 
 
 def test_identity():

tests/st/optimizer/test_asgd_cpu.py

@@ -45,7 +45,7 @@ def test_no_default_asgd_graph():
         no_default_fc1_bias_asgd, no_default_fc2_weight_asgd, no_default_fc2_bias_asgd
     context.set_context(mode=context.GRAPH_MODE, device_target='CPU')
     config = {'name': 'ASGD', 'lr': 0.001, 'lambd': 1e-3, 'alpha': 0.8, 't0': 50., 'weight_decay': 0.001}
-    loss, cells = build_network(config, is_group=True)
+    loss, cells = build_network(config)
     assert np.allclose(cells.ax[0].asnumpy(), no_default_fc1_weight_asgd, atol=1.e-5)
     assert np.allclose(cells.ax[1].asnumpy(), no_default_fc1_bias_asgd, atol=1.e-5)
     assert np.allclose(cells.ax[2].asnumpy(), no_default_fc2_weight_asgd, atol=1.e-5)

tests/ut/python/nn/optim/test_asgd.py

@@ -0,0 +1,102 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+""" test ASGD """
+import numpy as np
+import pytest
+
+import mindspore.nn as nn
+from mindspore import Tensor, Parameter
+from mindspore.nn.optim import ASGD
+from mindspore.ops import operations as P
+
+class Net(nn.Cell):
+    """ Net definition """
+
+    def __init__(self):
+        super(Net, self).__init__()
+        self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight")
+        self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias")
+        self.matmul = P.MatMul()
+        self.biasAdd = P.BiasAdd()
+
+    def construct(self, x):
+        x = self.biasAdd(self.matmul(x, self.weight), self.bias)
+        return x
+
+
+class NetWithoutWeight(nn.Cell):
+    def __init__(self):
+        super(NetWithoutWeight, self).__init__()
+        self.matmul = P.MatMul()
+
+    def construct(self, x):
+        x = self.matmul(x, x)
+        return x
+
+
+def test_asgdwithoutparam():
+    """
+    Feature: Test ASGD optimizer.
+    Description: Test if error is raised when trainable_params is empty.
+    Expectation: ValueError is raised.
+    """
+    net = NetWithoutWeight()
+    net.set_train()
+    with pytest.raises(ValueError, match=r"Optimizer got an empty parameters list"):
+        ASGD(net.trainable_params(), learning_rate=0.1)
+
+
+def test_asgd_lambd():
+    """
+    Feature: Test ASGD optimizer.
+    Description: Test if error is raised when the type of lambd  is not correct.
+    Expectation: ValueError is raised.
+    """
+    net = Net()
+    with pytest.raises(TypeError):
+        ASGD(net.get_parameters(), lambd=1, learning_rate=0.1)
+
+
+def test_asgd_alpha():
+    """
+    Feature: Test ASGD optimizer.
+    Description: Test if error is raised when the type of alpha  is not correct.
+    Expectation: ValueError is raised.
+    """
+    net = Net()
+    with pytest.raises(TypeError):
+        ASGD(net.get_parameters(), alpha=1, learning_rate=0.1)
+
+
+def test_asgd_t0():
+    """
+    Feature: Test ASGD optimizer.
+    Description: Test if error is raised when the type of t0  is not correct.
+    Expectation: ValueError is raised.
+    """
+    net = Net()
+    with pytest.raises(TypeError):
+        ASGD(net.get_parameters(), t0=1, learning_rate=0.1)
+
+
+def test_asgd_mindspore_with_empty_params():
+    """
+    Feature: Test ASGD optimizer.
+    Description: Test if error is raised when there is no trainable_params.
+    Expectation: ValueError is raised.
+    """
+    net = nn.Flatten()
+    with pytest.raises(ValueError, match=r"Optimizer got an empty parameters list"):
+        ASGD(net.get_parameters())

tests/ut/python/nn/optim/test_rprop.py

@@ -0,0 +1,125 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+""" test Rprop """
+import numpy as np
+import pytest
+
+import mindspore.nn as nn
+from mindspore import Tensor, Parameter
+from mindspore.nn.optim import Rprop
+from mindspore.ops import operations as P
+
+class Net(nn.Cell):
+    """ Net definition """
+
+    def __init__(self):
+        super(Net, self).__init__()
+        self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight")
+        self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias")
+        self.matmul = P.MatMul()
+        self.biasAdd = P.BiasAdd()
+
+    def construct(self, x):
+        x = self.biasAdd(self.matmul(x, self.weight), self.bias)
+        return x
+
+
+class NetWithoutWeight(nn.Cell):
+    def __init__(self):
+        super(NetWithoutWeight, self).__init__()
+        self.matmul = P.MatMul()
+
+    def construct(self, x):
+        x = self.matmul(x, x)
+        return x
+
+
+def test_rpropwithoutparam():
+    """
+    Feature: Test Rprop optimizer.
+    Description: Test if error is raised when trainable_params is empty.
+    Expectation: ValueError is raised.
+    """
+    net = NetWithoutWeight()
+    net.set_train()
+    with pytest.raises(ValueError, match=r"Optimizer got an empty parameters list"):
+        Rprop(net.trainable_params(), learning_rate=0.1)
+
+
+def test_rprop_tuple():
+    """
+    Feature: Test Rprop optimizer.
+    Description: Test if error is raised when the type of etas  and step_sizes is not correct.
+    Expectation: TypeError is raised.
+    """
+    net = Net()
+    with pytest.raises(TypeError):
+        Rprop(net.get_parameters(), etas=[0.5, 1.2], learning_rate=0.1)
+
+    with pytest.raises(TypeError):
+        Rprop(net.get_parameters(), step_sizes=[1e-6, 50.], learning_rate=0.1)
+
+
+def test_rprop_size():
+    """
+    Feature: Test Rprop optimizer.
+    Description: Test if error is raised when the size of etas  and step_sizes is not correct.
+    Expectation: ValueError is raised.
+    """
+    net = Net()
+    with pytest.raises(ValueError):
+        Rprop(net.get_parameters(), etas=(0.5, 1.2, 1.3), learning_rate=0.1)
+
+    with pytest.raises(ValueError):
+        Rprop(net.get_parameters(), step_sizes=(1e-6, 50., 60.), learning_rate=0.1)
+
+
+def test_rprop_stepsize():
+    """
+    Feature: Test Rprop optimizer.
+    Description: Test if error is raised when the value of step_sizes is not correct.
+    Expectation: ValueError is raised.
+    """
+    net = Net()
+    with pytest.raises(ValueError):
+        Rprop(net.get_parameters(), step_sizes=(50., 1e-6), learning_rate=0.1)
+
+
+def test_rprop_etas():
+    """
+    Feature: Test Rprop optimizer.
+    Description: Test if error is raised when the value range of etas is not correct.
+    Expectation: ValueError is raised.
+    """
+    net = Net()
+    with pytest.raises(ValueError):
+        Rprop(net.get_parameters(), etas=(0.5, 0.9), learning_rate=0.1)
+
+    with pytest.raises(ValueError):
+        Rprop(net.get_parameters(), etas=(1., 1.2), learning_rate=0.1)
+
+    with pytest.raises(ValueError):
+        Rprop(net.get_parameters(), etas=(-0.1, 1.2), learning_rate=0.1)
+
+
+def test_rprop_mindspore_with_empty_params():
+    """
+    Feature: Test Rprop optimizer.
+    Description: Test if error is raised when there is no trainable_params.
+    Expectation: ValueError is raised.
+    """
+    net = nn.Flatten()
+    with pytest.raises(ValueError, match=r"Optimizer got an empty parameters list"):
+        Rprop(net.get_parameters())

tests/ut/python/utils/test_initializer.py

@@ -56,7 +56,12 @@ def _check_uniform(tensor, boundary_a, boundary_b):
     return p > 0.0001
 
 
-def test_init_Initializer():
+def test_init_initializer():
+    """
+    Feature: Test initializer.
+    Description: Test initializer.
+    Expectation: Shape and value is initialized successfully..
+    """
     tensor = init.initializer(InitTwo(), [2, 2], ms.int32)
     assert tensor.shape == (2, 2)
     _check_value(tensor.init_data(), 2, 2)
@@ -203,6 +208,76 @@ def test_init_he_uniform_error():
         init.initializer(init.HeUniform(), [6], ms.float32).init_data()
 
 
+def test_init_identity():
+    """
+    Feature: Test identity initializer.
+    Description: Test if error is raised when the shape of the initialized tensor is not correct.
+    Expectation: ValueError is raised.
+    """
+    with py.raises(ValueError):
+        tensor = init.initializer(init.Identity(), [5, 4, 6], ms.float32)
+        tensor.init_data()
+
+
+def test_init_sparse():
+    """
+    Feature: Test sparse initializer.
+    Description: Test if error is raised when the shape of the initialized tensor is not correct.
+    Expectation: ValueError is raised.
+    """
+    with py.raises(ValueError):
+        tensor = init.initializer(init.Sparse(sparsity=0.1), [5, 4, 6], ms.float32)
+        tensor.init_data()
+
+
+def test_init_dirac():
+    """
+    Feature: Test dirac initializer.
+    Description: Test if error is raised when the shape of the initialized tensor is not correct.
+    or shape[0] is not divisible by group.
+    Expectation: ValueError is raised.
+    """
+    with py.raises(ValueError):
+        tensor1 = init.initializer(init.Dirac(groups=2), [5, 4, 6], ms.float32)
+        tensor1.init_data()
+
+    with py.raises(ValueError):
+        tensor2 = init.initializer(init.Dirac(groups=1), [5, 4], ms.float32)
+        tensor2.init_data()
+
+    with py.raises(ValueError):
+        tensor3 = init.initializer(init.Dirac(groups=1), [5, 4, 6, 7, 8, 9], ms.float32)
+        tensor3.init_data()
+
+
+def test_init_orthogonal():
+    """
+    Feature: Test orthogonal initializer.
+    Description: Test if error is raised when the shape of the initialized tensor is not correct.
+    Expectation: ValueError is raised.
+    """
+    with py.raises(ValueError):
+        tensor = init.initializer(init.Orthogonal(), [5,], ms.float32)
+        tensor.init_data()
+
+
+def test_init_variancescaling():
+    """
+    Feature: Test orthogonal initializer.
+    Description: Test if error is raised when scale is less than 0 or mode and distribution are not correct.
+    Expectation: ValueError is raised.
+    """
+    with py.raises(ValueError):
+        init.initializer(init.VarianceScaling(scale=-0.1), [5, 4, 6], ms.float32)
+
+    with py.raises(ValueError):
+        init.initializer(init.VarianceScaling(scale=0.1, mode='fans'), [5, 4, 6], ms.float32)
+
+    with py.raises(ValueError):
+        init.initializer(init.VarianceScaling(scale=0.1, mode='fan_in',
+                                              distribution='uniformal'), [5, 4, 6], ms.float32)
+
+
 def test_conv2d_abnormal_kernel_negative():
     kernel = np.random.randn(64, 3, 7, 7).astype(np.float32)
     with py.raises(ValueError):