add FusedSparse(Adam/LazyAdam/Ftrl/ProximalAdagrad) for aicpu

mindspore/ops/_op_impl/aicpu/__init__.py

@@ -44,3 +44,7 @@ from .laplace import _laplace_aicpu
 from .strided_slice import _strided_slice_aicpu
 from .strided_slice_grad import _strided_slice_grad_aicpu
 from .end_of_sequence import _end_of_sequence_aicpu
+from .fused_sparse_adam import _fused_sparse_adam_aicpu
+from .fused_sparse_lazy_adam import _fused_sparse_lazy_adam_aicpu
+from .fused_sparse_ftrl import _fused_sparse_ftrl_aicpu
+from .fused_sparse_proximal_adagrad import _fused_sparse_proximal_adagrad_aicpu

mindspore/ops/_op_impl/aicpu/fused_sparse_adam.py

@@ -0,0 +1,46 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""FusedSparseAdam op"""
+from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType
+
+fused_sparse_adam_op_info = AiCPURegOp("FusedSparseAdam") \
+    .fusion_type("OPAQUE") \
+    .attr("use_locking", "bool") \
+    .attr("use_nesterov", "bool") \
+    .input(0, "var", "required") \
+    .input(1, "m", "required") \
+    .input(2, "v", "required") \
+    .input(3, "beta1_power", "required") \
+    .input(4, "beta2_power", "required") \
+    .input(5, "lr", "required") \
+    .input(6, "beta1", "required") \
+    .input(7, "beta2", "required") \
+    .input(8, "epsilon", "required") \
+    .input(9, "grad", "required") \
+    .input(10, "indices", "required") \
+    .output(0, "var", "required") \
+    .output(1, "m", "required") \
+    .output(2, "v", "required") \
+    .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default) \
+    .get_op_info()
+
+@op_info_register(fused_sparse_adam_op_info)
+def _fused_sparse_adam_aicpu():
+    """FusedSparseAdam aicpu register"""
+    return

mindspore/ops/_op_impl/aicpu/fused_sparse_ftrl.py

@@ -0,0 +1,41 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""FusedSparseFtrl op"""
+from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType
+
+fused_sparse_ftrl_op_info = AiCPURegOp("FusedSparseFtrl") \
+    .fusion_type("OPAQUE") \
+    .attr("lr", "float") \
+    .attr("l1", "float") \
+    .attr("l2", "float") \
+    .attr("lr_power", "float") \
+    .attr("use_locking", "bool") \
+    .input(0, "var", "required") \
+    .input(1, "accum", "required") \
+    .input(2, "linear", "required") \
+    .input(3, "grad", "required") \
+    .input(4, "indices", "required") \
+    .output(0, "var", "required") \
+    .output(1, "accum", "required") \
+    .output(2, "linear", "required") \
+    .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default,
+                  DataType.I32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \
+    .get_op_info()
+
+@op_info_register(fused_sparse_ftrl_op_info)
+def _fused_sparse_ftrl_aicpu():
+    """FusedSparseFtrl aicpu register"""
+    return

mindspore/ops/_op_impl/aicpu/fused_sparse_lazy_adam.py

@@ -0,0 +1,46 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""FusedSparseLazyAdam op"""
+from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType
+
+fused_sparse_lazy_adam_op_info = AiCPURegOp("FusedSparseLazyAdam") \
+    .fusion_type("OPAQUE") \
+    .attr("use_locking", "bool") \
+    .attr("use_nesterov", "bool") \
+    .input(0, "var", "required") \
+    .input(1, "m", "required") \
+    .input(2, "v", "required") \
+    .input(3, "beta1_power", "required") \
+    .input(4, "beta2_power", "required") \
+    .input(5, "lr", "required") \
+    .input(6, "beta1", "required") \
+    .input(7, "beta2", "required") \
+    .input(8, "epsilon", "required") \
+    .input(9, "grad", "required") \
+    .input(10, "indices", "required") \
+    .output(0, "var", "required") \
+    .output(1, "m", "required") \
+    .output(2, "v", "required") \
+    .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default) \
+    .get_op_info()
+
+@op_info_register(fused_sparse_lazy_adam_op_info)
+def _fused_sparse_lazy_adam_aicpu():
+    """FusedSparseLazyAdam aicpu register"""
+    return

mindspore/ops/_op_impl/aicpu/fused_sparse_proximal_adagrad.py

@@ -0,0 +1,39 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""FusedSparseProximalAdagrad op"""
+from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType
+
+fused_sparse_proximal_adagrad_op_info = AiCPURegOp("FusedSparseProximalAdagrad") \
+    .fusion_type("OPAQUE") \
+    .attr("use_locking", "bool") \
+    .input(0, "var", "required") \
+    .input(1, "accum", "required") \
+    .input(2, "lr", "required") \
+    .input(3, "l1", "required") \
+    .input(4, "l2", "required") \
+    .input(5, "grad", "required") \
+    .input(6, "indices", "required") \
+    .output(0, "var", "required") \
+    .output(1, "accum", "required") \
+    .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default,
+                  DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, DataType.F32_Default,
+                  DataType.F32_Default) \
+    .get_op_info()
+
+@op_info_register(fused_sparse_proximal_adagrad_op_info)
+def _fused_sparse_proximal_adagrad_aicpu():
+    """FusedSparseProximalAdagrad aicpu register"""
+    return

tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_adam.py

@@ -0,0 +1,53 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+import numpy as np
+import mindspore.nn as nn
+import mindspore.common.dtype as mstype
+import mindspore.context as context
+from mindspore import Tensor
+from mindspore.ops import operations as P
+from mindspore.common.parameter import Parameter
+
+context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
+
+beta1_power = 0.9
+beta2_power = 0.999
+lr = 0.001
+beta1 = 0.9
+beta2 = 0.999
+epsilon = 1e-8
+
+class Net(nn.Cell):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.fused_sparse_adam = P.FusedSparseAdam()
+        self.var = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="var")
+        self.m = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="m")
+        self.v = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="v")
+
+    def construct(self, grad, indices):
+        return self.fused_sparse_adam(self.var, self.m, self.v, beta1_power, beta2_power, lr, beta1, beta2, epsilon,
+                                      grad, indices)
+
+def test_net():
+    gradient = Tensor(np.array([0.22948648, 0.14569908, 0.92861906, 0.66870148])
+                      .reshape([2, 1, 2]).astype(np.float32))
+    indices = Tensor([0, 1], mstype.int32)
+    net = Net()
+    output = net(gradient, indices)
+    print(output)
+    print(net.var.default_input)
+    print(net.m.default_input)
+    print(net.v.default_input)

tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_ftrl.py

@@ -0,0 +1,50 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+import numpy as np
+import mindspore.common.dtype as mstype
+import mindspore.nn as nn
+import mindspore.context as context
+from mindspore import Tensor
+from mindspore.ops import operations as P
+from mindspore.common.parameter import Parameter
+
+context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
+
+lr = 0.01
+l1 = 0.0
+l2 = 0.0
+lr_power = -0.5
+
+class Net(nn.Cell):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.fused_sparse_ftrl = P.FusedSparseFtrl(lr=0.1, l1=0.0, l2=0.0, lr_power=-0.5)
+        self.var = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="var")
+        self.accum = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="accum")
+        self.linear = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="linear")
+
+    def construct(self, grad, indices):
+        return self.fused_sparse_ftrl(self.var, self.accum, self.linear, grad, indices)
+
+def test_net():
+    gradient = Tensor(np.array([-3, 2, 3, 0, 0, 0, -4, -1, -2])
+                      .reshape([3, 3]).astype(np.float32))
+    indices = Tensor(np.ones([3]), mstype.int32)
+    net = Net()
+    output = net(gradient, indices)
+    print(output)
+    print(net.var.default_input)
+    print(net.accum.default_input)
+    print(net.linear.default_input)

tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_lazy_adam.py

@@ -0,0 +1,53 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+import numpy as np
+import mindspore.common.dtype as mstype
+import mindspore.context as context
+import mindspore.nn as nn
+from mindspore import Tensor
+from mindspore.ops import operations as P
+from mindspore.common.parameter import Parameter
+
+context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
+
+beta1_power = 0.9
+beta2_power = 0.999
+lr = 0.001
+beta1 = 0.9
+beta2 = 0.999
+epsilon = 1e-8
+
+class Net(nn.Cell):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.fused_sparse_lazy_adam = P.FusedSparseLazyAdam()
+        self.var = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="var")
+        self.m = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="m")
+        self.v = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="v")
+
+    def construct(self, grad, indices):
+        return self.fused_sparse_lazy_adam(self.var, self.m, self.v, beta1_power, beta2_power,
+                                           lr, beta1, beta2, epsilon, grad, indices)
+
+def test_net():
+    gradient = Tensor(np.array([0.22948648, 0.14569908, 0.92861906, 0.66870148])
+                      .reshape([2, 1, 2]).astype(np.float32))
+    indices = Tensor([0, 1], mstype.int32)
+    net = Net()
+    output = net(gradient, indices)
+    print(output)
+    print(net.var.default_input)
+    print(net.m.default_input)
+    print(net.v.default_input)

tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_proximal_adagrad.py

@@ -0,0 +1,47 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+import numpy as np
+import mindspore.nn as nn
+import mindspore.context as context
+import mindspore.common.dtype as mstype
+from mindspore import Tensor
+from mindspore.ops import operations as P
+from mindspore.common.parameter import Parameter
+
+context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
+
+class Net(nn.Cell):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.fused_sparse_proximal_adagrad = P.FusedSparseProximalAdagrad()
+        self.var = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="var")
+        self.accum = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="accum")
+        self.lr = 0.01
+        self.l1 = 0.0
+        self.l2 = 0.0
+
+    def construct(self, grad, indices):
+        return self.fused_sparse_proximal_adagrad(self.var, self.accum, self.lr, self.l1, self.l2,
+                                                  grad, indices)
+
+def test_net():
+    gradient = Tensor(np.array([-3, 2, 3, 0, 0, 0, -4, -1, -2])
+                      .reshape([3, 3]).astype(np.float32))
+    indices = Tensor(np.ones([3]), mstype.int32)
+    net = Net()
+    output = net(gradient, indices)
+    print(output)
+    print(net.var.default_input)
+    print(net.accum.default_input)