From c713382798faaab5a29d005741c6b61a81bd27cd Mon Sep 17 00:00:00 2001 From: wuxuejian Date: Tue, 7 Jul 2020 20:15:16 +0800 Subject: [PATCH 01/13] add Padding op --- mindspore/ops/_op_impl/aicpu/__init__.py | 1 + mindspore/ops/_op_impl/aicpu/padding.py | 41 +++++++++++++++++++ mindspore/ops/operations/__init__.py | 3 +- mindspore/ops/operations/array_ops.py | 40 ++++++++++++++++++ tests/st/ops/ascend/test_drop_out_gen_mask.py | 2 +- tests/st/ops/ascend/test_padding.py | 40 ++++++++++++++++++ 6 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 mindspore/ops/_op_impl/aicpu/padding.py create mode 100644 tests/st/ops/ascend/test_padding.py diff --git a/mindspore/ops/_op_impl/aicpu/__init__.py b/mindspore/ops/_op_impl/aicpu/__init__.py index 8eb08aea2f..5551dc58b4 100644 --- a/mindspore/ops/_op_impl/aicpu/__init__.py +++ b/mindspore/ops/_op_impl/aicpu/__init__.py @@ -15,6 +15,7 @@ """aicpu ops""" from .init_data_set_queue import _init_data_set_queue_aicpu from .embedding_lookup import _embedding_lookup_aicpu +from .padding import _padding_aicpu from .dropout_genmask import _dropout_genmask_aicpu from .get_next import _get_next_aicpu from .print_tensor import _print_aicpu diff --git a/mindspore/ops/_op_impl/aicpu/padding.py b/mindspore/ops/_op_impl/aicpu/padding.py new file mode 100644 index 0000000000..4a67376fbd --- /dev/null +++ b/mindspore/ops/_op_impl/aicpu/padding.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Padding op""" +from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType + +padding_op_info = AiCPURegOp("Padding") \ + .fusion_type("OPAQUE") \ + .input(0, "x", "required") \ + .output(0, "y", "required") \ + .attr("pad_dim_size", "int") \ + .dtype_format(DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.I16_Default, DataType.I16_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I64_Default, DataType.I64_Default) \ + .dtype_format(DataType.U8_Default, DataType.U8_Default) \ + .dtype_format(DataType.U16_Default, DataType.U16_Default) \ + .dtype_format(DataType.U32_Default, DataType.U32_Default) \ + .dtype_format(DataType.U64_Default, DataType.U64_Default) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F64_Default, DataType.F64_Default) \ + .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ + .get_op_info() + +@op_info_register(padding_op_info) +def _padding_aicpu(): + """Padding AiCPU register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 1602f2594d..b575718668 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -27,7 +27,7 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, Rank, Reshape, ResizeNearestNeighbor, ArgMinWithValue, SameTypeShape, ScatterAdd, ScatterSub, ScatterMul, ScatterDiv, ScatterMax, ScatterMin, ScatterUpdate, ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, - Shape, Size, Slice, Split, TransShape, ParallelConcat, + Shape, Size, Slice, Split, TransShape, ParallelConcat, Padding, Squeeze, StridedSlice, Tile, TensorScatterUpdate, Transpose, TruncatedNormal, TupleToArray, UnsortedSegmentMin, UnsortedSegmentProd, UnsortedSegmentSum, SpaceToDepth, DepthToSpace, SpaceToBatch, BatchToSpace, @@ -137,6 +137,7 @@ __all__ = [ 'GatherV2', 'SparseGatherV2', 'EmbeddingLookup', + 'Padding', 'Concat', 'Pack', 'Unpack', diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 1e28a56db1..918ad6e0e6 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -602,6 +602,46 @@ class SparseGatherV2(GatherV2): """ +class Padding(PrimitiveWithInfer): + """ + Extend the last dimension of input tensor from 1 to pad_dim_size, fill with 0. + + Args: + pad_dim_size (int): The extend value of last dimension of x, must be positive. + + Inputs: + - **x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. The rank of x should be at least 2. + The last dimension of x should be 1. + + Outputs: + Tensor, the shape of tensor is :math:`(z_1, z_2, ..., z_N)`. + + Examples: + >>> x = Tensor(np.array([[8], [10]]), mindspore.float32) + >>> pad_dim_size = 4 + >>> out = P.Padding(pad_dim_size)(x) + [[8, 0, 0, 0], [10, 0, 0, 0]] + """ + @prim_attr_register + def __init__(self, pad_dim_size=8): + """init padding""" + validator.check_value_type("pad_dim_size", pad_dim_size, [int], self.name) + validator.check_integer("pad_dim_size", pad_dim_size, 0, Rel.GT, self.name) + self.pad_dim_size = pad_dim_size + + def __infer__(self, x): + validator.check_subclass("x", x['dtype'], mstype.tensor, self.name) + x_shape = list(x['shape']) + validator.check_integer("rank of x", len(x_shape), 1, Rel.GT, self.name) + validator.check_integer("last dim of x", x_shape[-1], 1, Rel.EQ, self.name) + out_shape = x_shape + out_shape[-1] = self.pad_dim_size + out = {'shape': out_shape, + 'dtype': x['dtype'], + 'value': None} + return out + + class Split(PrimitiveWithInfer): """ Splits input tensor into output_num of tensors along the given axis and output numbers. diff --git a/tests/st/ops/ascend/test_drop_out_gen_mask.py b/tests/st/ops/ascend/test_drop_out_gen_mask.py index 6771a3a68b..58a37b495c 100644 --- a/tests/st/ops/ascend/test_drop_out_gen_mask.py +++ b/tests/st/ops/ascend/test_drop_out_gen_mask.py @@ -43,4 +43,4 @@ def test_net(): tx, ty = Tensor(x), Tensor(y) output = mask(tx, ty) print(output.asnumpy()) - assert ([255, 255, 255, 255] == output.asnumpy()).all() + assert ([255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255] == output.asnumpy()).all() diff --git a/tests/st/ops/ascend/test_padding.py b/tests/st/ops/ascend/test_padding.py new file mode 100644 index 0000000000..8a8ed19af3 --- /dev/null +++ b/tests/st/ops/ascend/test_padding.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np + +import mindspore.context as context +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor +from mindspore.ops import operations as P + +context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, pad_dim_size): + super(Net, self).__init__() + self.padding = P.Padding(pad_dim_size) + + def construct(self, x): + return self.padding(x) + + +def test_padding(): + x = Tensor(np.array([[8], [10]]), mstype.int32) + padding = Net(4) + out = padding(x) + assert(out.asnumpy() == [[8, 0, 0, 0], [10, 0, 0, 0]]).all() From 16c38f2897a430015399e465fdc670d49d94493c Mon Sep 17 00:00:00 2001 From: liuxiao93 Date: Fri, 17 Jul 2020 11:48:18 +0800 Subject: [PATCH 02/13] change ApplyMomentumD->ApplyMoment for GE. --- mindspore/ccsrc/transform/graph_ir/convert.cc | 2 +- mindspore/ccsrc/transform/graph_ir/op_declare.cc | 10 +++++----- mindspore/ccsrc/transform/graph_ir/op_declare.h | 4 ++-- mindspore/ops/operations/nn_ops.py | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mindspore/ccsrc/transform/graph_ir/convert.cc b/mindspore/ccsrc/transform/graph_ir/convert.cc index 7419dd2cc9..1978181fec 100644 --- a/mindspore/ccsrc/transform/graph_ir/convert.cc +++ b/mindspore/ccsrc/transform/graph_ir/convert.cc @@ -216,7 +216,7 @@ std::unordered_map &DfGraphConvertor::get_adpt_ma {string(kNameIOU), ADPT_DESC(Iou)}, {string(kNameGreaterEqual), ADPT_DESC(GreaterEqual)}, {string(kNameSlice), ADPT_DESC(SliceD)}, - {string(kNameApplyMomentum), ADPT_DESC(ApplyMomentumD)}, + {string(kNameApplyMomentum), ADPT_DESC(ApplyMomentum)}, {string(kNameMaxPool), ADPT_DESC(MaxPool)}, {string(kNameAvgPool), ADPT_DESC(AvgPool)}, {string(kNameMaxPoolWithArgmax), ADPT_DESC(MaxPoolWithArgmax)}, diff --git a/mindspore/ccsrc/transform/graph_ir/op_declare.cc b/mindspore/ccsrc/transform/graph_ir/op_declare.cc index e3751e0c92..b42519900e 100644 --- a/mindspore/ccsrc/transform/graph_ir/op_declare.cc +++ b/mindspore/ccsrc/transform/graph_ir/op_declare.cc @@ -143,12 +143,12 @@ INPUT_MAP(Constant) = EMPTY_INPUT_MAP; ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; -// ApplyMomentumD -INPUT_MAP(ApplyMomentumD) = { +// ApplyMomentum +INPUT_MAP(ApplyMomentum) = { {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}, {5, INPUT_DESC(momentum)}}; -ATTR_MAP(ApplyMomentumD) = {{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}, - {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; -OUTPUT_MAP(ApplyMomentumD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +ATTR_MAP(ApplyMomentum) = {{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ApplyMomentum) = {{0, OUTPUT_DESC(var)}}; // ScalarSummary INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; diff --git a/mindspore/ccsrc/transform/graph_ir/op_declare.h b/mindspore/ccsrc/transform/graph_ir/op_declare.h index e493ea0e52..b849461d56 100755 --- a/mindspore/ccsrc/transform/graph_ir/op_declare.h +++ b/mindspore/ccsrc/transform/graph_ir/op_declare.h @@ -334,8 +334,8 @@ DECLARE_OP_ADAPTER(Assign) DECLARE_OP_USE_OUTPUT(Assign) DECLARE_OP_ADAPTER(Constant) DECLARE_OP_USE_OUTPUT(Constant) -DECLARE_OP_ADAPTER(ApplyMomentumD) -DECLARE_OP_USE_OUTPUT(ApplyMomentumD) +DECLARE_OP_ADAPTER(ApplyMomentum) +DECLARE_OP_USE_OUTPUT(ApplyMomentum) // ** Summary Operations ** DECLARE_OP_ADAPTER(Summary) diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index a9bdf07d28..83e6454a00 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -1615,9 +1615,10 @@ class ApplyMomentum(PrimitiveWithInfer): self.init_prim_io_names(inputs=['variable', 'accumulation', 'learning_rate', 'gradient', 'momentum'], outputs=['output']) self.is_tbe = context.get_context("device_target") == "Ascend" + self.is_ge = context.get_context("enable_ge") def infer_shape(self, v_shape, a_shape, l_shape, g_shape, m_shape): - if self.is_tbe: + if not self.is_ge and self.is_tbe: return v_shape, v_shape return v_shape @@ -1629,7 +1630,7 @@ class ApplyMomentum(PrimitiveWithInfer): validator.check_scalar_or_tensor_type_same({"l_dtype": l_dtype}, valid_types, self.name) validator.check_scalar_or_tensor_type_same({"g_dtype": g_dtype}, valid_types, self.name) validator.check_scalar_or_tensor_type_same({"m_dtype": m_dtype}, valid_types, self.name) - if self.is_tbe: + if not self.is_ge and self.is_tbe: return g_dtype, g_dtype return g_dtype From b54625b938fa51516dc9887963c9b2118d7fa7bf Mon Sep 17 00:00:00 2001 From: changzherui Date: Tue, 21 Jul 2020 19:24:02 +0800 Subject: [PATCH 03/13] modify sync code --- .../parallel/graph_util/generate_graph.cc | 8 +- mindspore/ccsrc/operator/prim_structures.cc | 707 ------------------ .../ccsrc/optimizer/irpass/ref_eliminate.h | 93 --- .../parallel/graph_util/generate_graph.cc | 175 ----- mindspore/ops/operations/__init__.py | 3 - 5 files changed, 4 insertions(+), 982 deletions(-) delete mode 100644 mindspore/ccsrc/operator/prim_structures.cc delete mode 100644 mindspore/ccsrc/optimizer/irpass/ref_eliminate.h delete mode 100644 mindspore/ccsrc/parallel/graph_util/generate_graph.cc diff --git a/mindspore/ccsrc/frontend/parallel/graph_util/generate_graph.cc b/mindspore/ccsrc/frontend/parallel/graph_util/generate_graph.cc index 30c25e5f26..7813332765 100644 --- a/mindspore/ccsrc/frontend/parallel/graph_util/generate_graph.cc +++ b/mindspore/ccsrc/frontend/parallel/graph_util/generate_graph.cc @@ -31,13 +31,13 @@ std::string GetOpPythonPath(const OperatorName &op_name) { const std::string inner_ops_module = INNER_OP_PATH; py::module mod = py::module::import(common::SafeCStr(ops_module)); py::module inner_mod = py::module::import(common::SafeCStr(inner_ops_module)); - if (!py::hasattr(mod, common::SafeCStr(op_name))) { - if (!py::hasattr(inner_mod, common::SafeCStr(op_name))) { + if (!py::hasattr(inner_mod, common::SafeCStr(op_name))) { + if (!py::hasattr(mod, common::SafeCStr(op_name))) { MS_LOG(EXCEPTION) << ops_module << " or " << inner_ops_module << " don't have op:" << op_name; } - return inner_ops_module; + return ops_module; } - return ops_module; + return inner_ops_module; } ValuePtr CreatOpInstance(const OperatorAttrs &attrs, const OperatorName &op_name, const std::string &instance_name) { diff --git a/mindspore/ccsrc/operator/prim_structures.cc b/mindspore/ccsrc/operator/prim_structures.cc deleted file mode 100644 index 68215d5e58..0000000000 --- a/mindspore/ccsrc/operator/prim_structures.cc +++ /dev/null @@ -1,707 +0,0 @@ -/** - * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). - * - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pipeline/static_analysis/prim.h" -#include "pipeline/static_analysis/utils.h" -#include "pipeline/static_analysis/param_validator.h" -#include "operator/ops.h" -#include "utils/convert_utils.h" -#include "ir/tensor_py.h" - -using mindspore::tensor::TensorPy; - -namespace mindspore { -namespace abstract { - -AbstractBasePtr InferImplStringEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: two scalars whose value is a string. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractScalarPtr scalar_x = CheckArg(op_name, args_spec_list, 0); - AbstractScalarPtr scalar_y = CheckArg(op_name, args_spec_list, 1); - - ValuePtr value_x = scalar_x->BuildValue(); - ValuePtr value_y = scalar_y->BuildValue(); - if (!value_x->isa() || !value_y->isa()) { - MS_LOG(EXCEPTION) << op_name << " requires 2 parameters are string, but got param0: " << value_x->ToString() - << ", param1: " << value_y->ToString(); - } - - bool ret = (value_x->cast()->value() == value_y->cast()->value()); - return std::make_shared(ret); -} - -AbstractBasePtr InferImplStringConcat(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: two scalars whose value is a string. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractScalarPtr scalar_x = CheckArg(op_name, args_spec_list, 0); - AbstractScalarPtr scalar_y = CheckArg(op_name, args_spec_list, 1); - - ValuePtr value_x = scalar_x->BuildValue(); - ValuePtr value_y = scalar_y->BuildValue(); - if (!value_x->isa() || !value_y->isa()) { - MS_LOG(EXCEPTION) << op_name << " requires 2 parameters are string, but got param0: " << value_x->ToString() - << ", param1: " << value_y->ToString(); - } - - std::string ret = (value_x->cast()->value() + value_y->cast()->value()); - return std::make_shared(ret); -} - -AbstractBasePtr InferImplMakeTuple(const AnalysisEnginePtr &, const PrimitivePtr &, - const AbstractBasePtrList &args_spec_list) { - return std::make_shared(args_spec_list); -} - -AbstractBasePtr InferImplMakeList(const AnalysisEnginePtr &, const PrimitivePtr &, - const AbstractBasePtrList &args_spec_list) { - return std::make_shared(args_spec_list); -} - -AbstractBasePtr InferImplMakeDict(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: two tuples. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractTuplePtr keys = CheckArg(op_name, args_spec_list, 0); - AbstractTuplePtr values = CheckArg(op_name, args_spec_list, 1); - - size_t keys_size = keys->size(); - if (values->size() != keys_size) { - MS_LOG(EXCEPTION) << op_name << " evaluator keys' size is not equal with values' size"; - } - - std::vector key_value; - AbstractScalarPtr key; - AbstractBasePtrList key_list = keys->elements(); - AbstractBasePtrList value_list = values->elements(); - for (size_t index = 0; index < keys_size; index++) { - key = CheckArg(op_name + "key", key_list, index); - ValuePtr keyPtr = key->BuildValue(); - MS_EXCEPTION_IF_NULL(keyPtr); - if (!keyPtr->isa()) { - MS_LOG(EXCEPTION) << op_name << " evaluator keys should be string, but got " << keyPtr->ToString(); - } - std::string key_string = GetValue(keyPtr); - key_value.emplace_back(key_string, value_list[index]); - } - return std::make_shared(key_value); -} - -AbstractBasePtr InferImplMakeKwarg(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a string and an object of a subclass of AbstractBase. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 0); - - ValuePtr keyPtr = key->BuildValue(); - if (!keyPtr->isa()) { - MS_LOG(EXCEPTION) << op_name << " evaluator key should be string, but got " << keyPtr->ToString(); - } - std::string key_string = GetValue(keyPtr); - return std::make_shared(key_string, args_spec_list[1]); -} - -AbstractBasePtr InferImplExtractKwarg(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a string and a keyword. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 0); - AbstractKeywordArgPtr kwarg = CheckArg(op_name, args_spec_list, 1); - - ValuePtr key_value = key->BuildValue(); - if (!key_value->isa()) { - MS_LOG(EXCEPTION) << op_name << " evaluator key should be string, but got " << key_value->ToString(); - } - std::string key_input = GetValue(key_value); - std::string key_actual = kwarg->get_key(); - if (key_actual != key_input) { - MS_LOG(EXCEPTION) << op_name << " evaluator input key should be same as AbstractKeywordArg' key, but input is " - << key_input << ", AbstractKeywordArg' key is " << key_actual; - } - return kwarg->get_arg(); -} - -AbstractBasePtr InferImplMakeSlice(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: three scalars whose value is an int32 number. - CheckArgsSize(primitive->name(), args_spec_list, 3); - size_t args_size = args_spec_list.size(); - for (size_t index = 0; index < args_size; index++) { - MS_EXCEPTION_IF_NULL(args_spec_list[index]); - if (!args_spec_list[index]->isa() && !args_spec_list[index]->isa()) { - MS_LOG(EXCEPTION) << "MakeSlice eval " << index << " parameter is neither AbstractScalar nor AbstractNone."; - } - if (args_spec_list[index]->isa() && - !dyn_cast(args_spec_list[index])->BuildValue()->isa()) { - MS_LOG(EXCEPTION) << "MakeSlice eval " << index << " parameter is an AbstractScalar, but is not an int32 number."; - } - } - // Slice: start, end, step - return std::make_shared(args_spec_list[0], args_spec_list[1], args_spec_list[2]); -} - -// Eval the return type of make_record -AbstractBasePtr InferImplMakeRecord(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: at lease two objects of a subclass of AbstractBase. - if (args_spec_list.size() < 2) { - MS_LOG(EXCEPTION) << "Typeof evaluator requires more than 1 parameter, while the input size is " - << args_spec_list.size() << "."; - } - - // args_spec_list[0] maybe AbstractScalarPtr or AbstractTypePtr - MS_EXCEPTION_IF_NULL(args_spec_list[0]); - TypePtr type = args_spec_list[0]->GetTypeTrack(); - MS_EXCEPTION_IF_NULL(type); - if (type->type_id() != kMetaTypeTypeType) { - MS_LOG(EXCEPTION) << "Can not make type(" << type->ToString() << ")not TypeType"; - } - - ValuePtr value_track = args_spec_list[0]->GetValueTrack(); - MS_EXCEPTION_IF_NULL(value_track); - TypePtr type_ptr = value_track->cast(); - if (type_ptr == nullptr) { - MS_LOG(EXCEPTION) << "Value type error, not Me type:" << value_track->ToString(); - } - - auto cls = dyn_cast(type_ptr); - MS_EXCEPTION_IF_NULL(cls); - ClassAttrVector attributes = cls->GetAttributes(); - CheckArgsSize(primitive->name(), args_spec_list, attributes.size() + 1); - - std::vector abs_attributes; - for (size_t i = 0; i < attributes.size(); i++) { - AbstractAttribute elem(attributes[i].first, args_spec_list[i + 1]); - abs_attributes.push_back(elem); - } - - return std::make_shared(cls->tag(), abs_attributes, cls->methods()); -} - -template -AbstractBasePtr InferTupleOrListGetItem(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { - // Inputs: a tuple or list and a scalar whose value is an int32 number. - CheckArgsSize(op_name, args_spec_list, 2); - auto queue = CheckArg(op_name, args_spec_list, 0); - AbstractScalarPtr index = CheckArg(op_name, args_spec_list, 1); - - ValuePtr index_value = index->BuildValue(); - if (!index_value->isa()) { - MS_EXCEPTION(IndexError) << op_name << " evaluator index should be an int32 number, but got " - << index_value->ToString(); - } - int idx_v = GetValue(index_value); - std::size_t nelems = queue->elements().size(); - if (idx_v >= SizeToInt(nelems) || idx_v < -SizeToInt(nelems)) { - MS_EXCEPTION(IndexError) << op_name << " evaluator index should be in range[-" << SizeToInt(nelems) << ", " - << SizeToInt(nelems) << "), but got " << idx_v << "."; - } - - std::size_t uidx_v = 0; - if (idx_v >= 0) { - uidx_v = IntToSize(idx_v); - } else { - uidx_v = IntToSize(idx_v + SizeToInt(nelems)); - } - return queue->elements()[uidx_v]; -} - -template -AbstractBasePtr InferTupleOrListSetItem(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { - // Inputs: a tuple or list, a scalar whose value is an int32 number and an object of a subclass of AbstractBase. - CheckArgsSize(op_name, args_spec_list, 3); - auto queue = CheckArg(op_name, args_spec_list, 0); - AbstractScalarPtr index = CheckArg(op_name, args_spec_list, 1); - - ValuePtr index_value = index->BuildValue(); - if (!index_value->isa()) { - MS_EXCEPTION(IndexError) << op_name << " evaluator index should be an int32 number, but got " - << index_value->ToString(); - } - int idx_v = GetValue(index_value); - if (idx_v < 0) { - MS_EXCEPTION(IndexError) << "The index of " << typeid(T).name() << " should be positive number, but got " << idx_v - << "."; - } - - size_t uidx_v = IntToSize(idx_v); - AbstractBasePtrList elements = queue->elements(); - std::size_t nelems = elements.size(); - if (uidx_v >= nelems) { - MS_EXCEPTION(IndexError) << op_name << " evaluator the index: " << uidx_v << " to set out of range: " << nelems - 1 - << "."; - } - elements[uidx_v] = args_spec_list[2]; - return std::make_shared(elements); -} - -AbstractBasePtr InferImplTupleGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListGetItem(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplListGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListGetItem(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplTupleSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListSetItem(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplListSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListSetItem(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplDictGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a dict and a scalar whose value is a string. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractDictionaryPtr dict = CheckArg(op_name, args_spec_list, 0); - AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 1); - - ValuePtr key_value = key->BuildValue(); - if (!key_value->isa()) { - MS_LOG(EXCEPTION) << op_name << " evaluator key should be string, but got " << key_value->ToString(); - } - auto key_str = GetValue(key_value); - std::vector dict_elems = dict->elements(); - auto it = std::find_if(dict_elems.begin(), dict_elems.end(), - [key_str](const AbstractAttribute &item) { return item.first == key_str; }); - - if (it == dict_elems.end()) { - MS_LOG(EXCEPTION) << "The key " << key_str << " does not exist in the dict:" << args_spec_list[0]->ToString(); - } - return it->second; -} - -AbstractBasePtr InferImplDictSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a dict and a scalar whose value is a string and an object of a subclass of AbstractBase. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 3); - AbstractDictionaryPtr dict = CheckArg(op_name, args_spec_list, 0); - AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 1); - - ValuePtr key_value = key->BuildValue(); - if (!key_value->isa()) { - MS_LOG(EXCEPTION) << op_name << " evaluator key should be string, but got " << key_value->ToString(); - } - std::string key_str = GetValue(key_value); - std::vector dict_elems = dict->elements(); - auto it = std::find_if(dict_elems.begin(), dict_elems.end(), - [key_str](AbstractAttribute &item) { return item.first == key_str; }); - - MS_EXCEPTION_IF_NULL(args_spec_list[2]); - auto new_ele = std::make_pair(key_str, args_spec_list[2]); - if (it != dict_elems.end()) { - int index = it - dict_elems.begin(); - dict_elems[IntToSize(index)] = new_ele; - } else { - dict_elems.push_back(new_ele); - } - return std::make_shared(dict_elems); -} - -AbstractBasePtr InferImplListAppend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a list and an object of a subclass of AbstractBase. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractListPtr list = CheckArg(op_name, args_spec_list, 0); - (void)AbstractJoin(list->elements()); - return list; -} - -template -AbstractBasePtr InferTupleOrListOrDictLen(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { - // Inputs: a tuple or list or dict. - CheckArgsSize(op_name, args_spec_list, 1); - auto arg = CheckArg(op_name, args_spec_list, 0); - return std::make_shared(SizeToInt(arg->size())); -} - -AbstractBasePtr InferImplTupleLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListOrDictLen(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplListLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListOrDictLen(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplDictLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferTupleOrListOrDictLen(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplArrayLen(const AnalysisEnginePtr &, const PrimitivePtr &, - const AbstractBasePtrList &args_spec_list) { - return std::make_shared(kAnyValue, kInt32); -} - -AbstractBasePtr InferImplListMap(const AnalysisEnginePtr &engine, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: fn, list1, list2, ... - MS_EXCEPTION_IF_NULL(engine); - if (args_spec_list.size() <= 1) { - MS_LOG(EXCEPTION) << "List_map requires at least 1 list. while the input size is " << args_spec_list.size() << "."; - } - AbstractFunctionPtr fn = CheckArg(primitive->name(), args_spec_list, 0); - // check args from 1. - CheckArgsSpec(AbstractBasePtrList(args_spec_list.begin() + 1, args_spec_list.end())); - - AbstractBasePtrList subargs; - for (std::size_t i = 1; i < args_spec_list.size(); i++) { - AbstractListPtr l_ptr = dyn_cast(args_spec_list[i]); - if (l_ptr == nullptr) { - MS_LOG(EXCEPTION) << "Argument[" << i << "] of list_map should be a list."; - } - subargs.push_back(AbstractJoin(l_ptr->elements())); - } - EvalResultPtr engin_exc = engine->Execute(fn, subargs); - AbstractBasePtrList result; - for (std::size_t i = 1; i < args_spec_list.size(); i++) { - result.push_back(engin_exc->abstract()); - } - return std::make_shared(result); -} - -AbstractBasePtr InferImplListReduce(const AnalysisEnginePtr &engine, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a fn, a list and an object of a subclass of a AbstractBase. - MS_EXCEPTION_IF_NULL(engine); - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 3); - AbstractFunctionPtr fn = CheckArg(op_name, args_spec_list, 0); - AbstractListPtr lst = CheckArg(op_name, args_spec_list, 1); - AbstractBasePtr dflt = args_spec_list[2]; - - AbstractBasePtr list_type = AbstractJoin(lst->elements()); - auto result1 = engine->Execute(fn, lst->elements()); - auto result2 = engine->Execute(fn, {dflt, list_type}); - MS_EXCEPTION_IF_NULL(result1->abstract()); - MS_EXCEPTION_IF_NULL(result2->abstract()); - return result1->abstract()->Join(result2->abstract()); -} - -AbstractBasePtr InferImplTupleReversed(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a tuple - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 1); - AbstractTuplePtr input = CheckArg(op_name, args_spec_list, 0); - - auto tuple_elements = input->elements(); - AbstractBasePtrList elem_list; - (void)std::transform(tuple_elements.rbegin(), tuple_elements.rend(), std::back_inserter(elem_list), - [](const AbstractBasePtr &elem) { return elem->Clone(); }); - return std::make_shared(elem_list); -} - -AbstractBasePtr DoInferReduceShape(const AbstractTuplePtr &x_shape, const ValuePtr &x_shp_value, - const ValueTuplePtr &axis_value_ptr, const PrimitivePtr &primitive) { - size_t x_rank = x_shape->size(); - std::set axis_set; - auto axis_data = axis_value_ptr->value(); - if (axis_data.empty()) { - int size = 1; - AbstractBasePtrList values(x_rank, std::make_shared(size)); - return std::make_shared(values); - } - - for (auto &elem : axis_data) { - int e_value = CheckAxis(primitive->name(), elem, -SizeToInt(x_rank), SizeToInt(x_rank) - 1); - (void)axis_set.insert(e_value); - } - - auto x_shp_data = x_shp_value->cast()->value(); - if (x_shp_data.size() < x_rank) { - MS_LOG(EXCEPTION) << "x_shape_data.size() " << x_shp_data.size() << " less than x_shape.size() " << x_rank; - } - AbstractBasePtrList values; - for (size_t i = 0; i < x_rank; i++) { - if (axis_set.count(SizeToInt(i)) || axis_set.count(SizeToInt(i) - SizeToInt(x_rank))) { - auto axis_v = MakeValue(1); - values.push_back(std::make_shared(axis_v, axis_v->type())); - } else { - int dim_value = x_shp_data[i]->cast()->value(); - auto dim = MakeValue(dim_value); - values.push_back(std::make_shared(dim, dim->type())); - } - } - - return std::make_shared(values); -} - -AbstractBasePtr InferImplReduceShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: x_shape, axis - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractTuplePtr shape_x = CheckArg(op_name, args_spec_list, 0); - MS_EXCEPTION_IF_NULL(args_spec_list[1]); - - auto x_shp_value = shape_x->BuildValue(); - if (x_shp_value->isa()) { - MS_LOG(EXCEPTION) << op_name - << " evaluator shape's data field can't be anything: " << args_spec_list[1]->ToString(); - } - - // Axis can be scalar, tuple or None - AbstractTuplePtr axis = nullptr; - if (args_spec_list[1]->isa()) { - MS_LOG(DEBUG) << op_name << " evaluator second parameter is scalar"; - AbstractBasePtrList axis_list = {dyn_cast(args_spec_list[1])}; - axis = std::make_shared(axis_list); - } else if (args_spec_list[1]->isa()) { - MS_LOG(DEBUG) << op_name << " evaluator second parameter is tuple"; - axis = args_spec_list[1]->cast(); - } else { - MS_LOG(EXCEPTION) << op_name << " evaluator second parameter should be a scalar or tuple, but got " - << args_spec_list[1]->ToString(); - } - - auto axis_value = axis->BuildValue(); - if (axis_value->isa()) { - MS_LOG(EXCEPTION) << op_name - << " evaluator shape's data field can't be anything: " << args_spec_list[1]->ToString(); - } - auto axis_value_ptr = axis_value->cast(); - MS_EXCEPTION_IF_NULL(axis_value_ptr); - - return DoInferReduceShape(shape_x, x_shp_value, axis_value_ptr, primitive); -} - -AbstractBasePtr InferImplTupleDiv(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: two tuples. - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 2); - AbstractTuplePtr shape_x = CheckArg(op_name, args_spec_list, 0); - AbstractTuplePtr div_shp = CheckArg(op_name, args_spec_list, 1); - MS_LOG(INFO) << "DivShape input:" << shape_x->ToString() << ", div:" << div_shp->ToString(); - - auto div_shp_value = div_shp->BuildValue(); - if (div_shp_value->isa()) { - MS_LOG(EXCEPTION) << "shape's data field can't be anythin: " << args_spec_list[0]->ToString(); - } - - auto shpx_value = shape_x->BuildValue(); - if (shpx_value->isa()) { - MS_LOG(EXCEPTION) << "shape's data field can't be anythin: " << args_spec_list[1]->ToString(); - } - - if (div_shp->size() != shape_x->size()) { - MS_LOG(EXCEPTION) << "tileshape elems shape must the same div_shp: " << div_shp->size() - << ", shapex: " << shape_x->size() << "."; - } - - auto shpx_data = shpx_value->cast()->value(); - auto div_shp_data = div_shp_value->cast()->value(); - AbstractBasePtrList values; - - for (size_t i = 0; i < div_shp_data.size(); i++) { - if (div_shp_data[i]->cast() == nullptr) { - MS_LOG(EXCEPTION) << "div_shp_shape data should be an int32 number, but it's " << args_spec_list[1]->ToString(); - } - int shapex_value = GetValue(shpx_data[i]); - int div_value = GetValue(div_shp_data[i]); - MS_LOG(DEBUG) << "div_shp_shape data shapex_value :" << shapex_value << " div_value: " << div_value; - if (div_value == 0) { - MS_LOG(EXCEPTION) << "error: division value should not be 0!"; - } - if ((shapex_value % div_value) != 0) { - MS_LOG(EXCEPTION) << "div_shp_shape data shapex must div int:" << shapex_value << " div_value: " << div_value; - } - - int result = shapex_value / div_value; - auto result_v = MakeValue(result); - values.push_back(std::make_shared(result_v, result_v->type())); - } - - return std::make_shared(values); -} - -AbstractBasePtr InferImplTuple2Array(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a tuple - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 1); - AbstractTuplePtr input = CheckArg(op_name, args_spec_list, 0); - - py::tuple data_tuple = ValuePtrToPyData(input->BuildValue()); - py::array data = py::array(data_tuple); - auto tensor = TensorPy::MakeTensor(data); - auto ret = tensor->ToAbstract(); - ret->set_value(tensor); - MS_LOG(DEBUG) << "Tuple2arry result AbstractTensor: " << ret->ToString(); - return ret; -} - -AbstractBasePtr InferImplShapeMul(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a tuple - // example: tuple = (1, 2, 3), shape_mul(tuple) = 1*2*3 = 6 - const std::string op_name = primitive->name(); - CheckArgsSize(op_name, args_spec_list, 1); - AbstractTuplePtr shape_x = CheckArg(op_name, args_spec_list, 0); - - auto shpx_value = shape_x->BuildValue(); - if (shpx_value->isa()) { - MS_LOG(EXCEPTION) << "shape's data field can't be anythin: " << shape_x->ToString(); - } - - auto shpx_data = shpx_value->cast()->value(); - - int result = 1; - for (size_t i = 0; i < shpx_data.size(); i++) { - int value = GetValue(shpx_data[i]); - result = IntMulWithOverflowCheck(result, value); - } - - auto result_v = MakeValue(result); - MS_LOG(DEBUG) << "shape mul result:" << result_v->ToString(); - return std::make_shared(result_v, result_v->type()); -} - -template -AbstractBasePtr InferImplTupleOrListEqual(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { - // Inputs: two tuples or two lists. - CheckArgsSize(op_name, args_spec_list, 2); - auto input_x = CheckArg(op_name, args_spec_list, 0); - auto input_y = CheckArg(op_name, args_spec_list, 1); - - ValuePtr x_value = input_x->BuildValue(); - ValuePtr y_value = input_y->BuildValue(); - return std::make_shared(*x_value == *y_value); -} - -AbstractBasePtr InferImplTupleEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferImplTupleOrListEqual(primitive->name(), args_spec_list); -} - -AbstractBasePtr InferImplListEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - return InferImplTupleOrListEqual(primitive->name(), args_spec_list); -} - -struct SlideInfo { - int start; - int step; - int stop; -}; - -void CalcSlidePara(const AbstractBasePtrList &args_spec_list, SlideInfo *slide) { - int arg1 = 0; - int arg2 = 0; - if (!args_spec_list.empty()) { - MS_EXCEPTION_IF_NULL(args_spec_list[0]); - auto arg_value = args_spec_list[0]->BuildValue(); - if (!arg_value->isa()) { - MS_LOG(EXCEPTION) << "Only supported input an int32 number."; - } - arg1 = GetValue(arg_value); - } - - if (args_spec_list.size() >= 2) { - MS_EXCEPTION_IF_NULL(args_spec_list[1]); - auto arg_value = args_spec_list[1]->BuildValue(); - if (!arg_value->isa()) { - MS_LOG(EXCEPTION) << "Only supported input an int32 number."; - } - arg2 = GetValue(arg_value); - } - - if (args_spec_list.size() == 3) { - MS_EXCEPTION_IF_NULL(args_spec_list[2]); - auto arg_value = args_spec_list[2]->BuildValue(); - if (!arg_value->isa()) { - MS_LOG(EXCEPTION) << "Only supported input an int32 number."; - } - slide->step = GetValue(arg_value); - slide->start = arg1; - slide->stop = arg2; - } - - if (args_spec_list.size() == 2) { - slide->start = arg1; - slide->stop = arg2; - } - - if (args_spec_list.size() == 1) { - slide->stop = arg1; - } -} - -AbstractBasePtr InferImplMakeRange(const AnalysisEnginePtr &, const PrimitivePtr &, - const AbstractBasePtrList &args_spec_list) { - if (args_spec_list.empty()) { - MS_LOG(EXCEPTION) << "Cannot make range from empty input."; - } - - if (args_spec_list.size() > 3) { - MS_LOG(EXCEPTION) << "Error args size of make range operational."; - } - - SlideInfo slide = {0, 1, 0}; - CalcSlidePara(args_spec_list, &slide); - - if (slide.step == 0) { - MS_LOG(EXCEPTION) << "Error, step value is 0."; - } - - AbstractBasePtrList args; - if (slide.start <= slide.stop) { - if (slide.step <= 0) { - MS_LOG(EXCEPTION) << "Error slice[" << slide.start << ", " << slide.stop << ", " << slide.step << "]"; - } - for (int i = slide.start; i < slide.stop; i += slide.step) { - args.push_back(abstract::FromValue(i)); - } - } else { - if (slide.step >= 0) { - MS_LOG(EXCEPTION) << "Error slice[" << slide.start << ", " << slide.stop << ", " << slide.step << "]"; - } - for (int i = slide.start; i > slide.stop; i += slide.step) { - args.push_back(abstract::FromValue(i)); - } - } - - return std::make_shared(args); -} - -AbstractBasePtr InferImplStopGradient(const AnalysisEnginePtr &, const PrimitivePtr &primitive, - const AbstractBasePtrList &args_spec_list) { - // Inputs: a tensor - CheckArgsSize(primitive->name(), args_spec_list, 1); - return args_spec_list[0]->Clone(); -} -} // namespace abstract -} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h deleted file mode 100644 index a9e4659eae..0000000000 --- a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REF_ELIMINATE_H_ -#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REF_ELIMINATE_H_ - -#include - -#include "ir/pattern_matcher.h" -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" - -namespace mindspore { -namespace opt { -namespace irpass { -// {prim::kPrimMakeRef, X, Y, Z} -> Y -class MakeRefEliminater : public OptimizerCaller { - public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - PatternNode x, y, z; - MATCH_REPLACE(node, PPrimitive(prim::kPrimMakeRef, x, y, z), y); - return nullptr; - } -}; - -// {prim::kPrimGetRefValue, Parameter} -> Parameter -// {prim::kPrimGetRefOrigin, Parameter} -> Parameter -class GetRefParamEliminater : public OptimizerCaller { - public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - PatternNode x; - MATCH_REPLACE_IF(node, PPrimitive(prim::kPrimGetRefValue, x), x, x.CheckFunc(IsParam, node)); - MATCH_REPLACE_IF(node, PPrimitive(prim::kPrimGetRefOrigin, x), x, x.CheckFunc(IsParam, node)); - return nullptr; - } -}; - -// {prim::kPrimGetRefKey, {prim::kPrimMakeRef, X, Y, Z}} -> X -// {prim::kPrimGetRefValue, {prim::kPrimMakeRef, X, Y, Z}} -> Y -// {prim::kPrimGetRefOrigin, {prim::kPrimMakeRef, X, Y, Z}} -> Z -class GetMakeRefEliminater : public OptimizerCaller { - public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - PatternNode x, y, z; - MATCH_REPLACE(node, PPrimitive(prim::kPrimGetRefKey, PPrimitive(prim::kPrimMakeRef, x, y, z)), x); - MATCH_REPLACE(node, PPrimitive(prim::kPrimGetRefValue, PPrimitive(prim::kPrimMakeRef, x, y, z)), y); - MATCH_REPLACE(node, PPrimitive(prim::kPrimGetRefOrigin, PPrimitive(prim::kPrimMakeRef, x, y, z)), z); - return nullptr; - } -}; - -// IsValueNode -class ReplaceRefkeyByParam : public OptimizerCaller { - public: - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { - auto RefKeyLambda = [&node, &optimizer]() -> AnfNodePtr { - auto refkey = GetValueNode(node); - auto resource = std::dynamic_pointer_cast(optimizer->resource()); - MS_EXCEPTION_IF_NULL(resource); - - auto top_graph = resource->func_graph(); - MS_EXCEPTION_IF_NULL(top_graph); - - for (const auto &tnode : top_graph->parameters()) { - auto para = tnode->cast(); - if (para != nullptr && para->name() == refkey->tag()) { - return para; - } - } - return nullptr; - }; - PatternNode x; - MATCH_REPLACE_LAMBDA_IF(node, x, RefKeyLambda, x.CheckFunc(IsValueNode, node)); - return nullptr; - } -}; -} // namespace irpass -} // namespace opt -} // namespace mindspore -#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REF_ELIMINATE_H_ diff --git a/mindspore/ccsrc/parallel/graph_util/generate_graph.cc b/mindspore/ccsrc/parallel/graph_util/generate_graph.cc deleted file mode 100644 index 4db912f63e..0000000000 --- a/mindspore/ccsrc/parallel/graph_util/generate_graph.cc +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "parallel/graph_util/generate_graph.h" - -#include -#include -#include -#include - -using mindspore::tensor::Tensor; - -namespace mindspore { -namespace parallel { -std::string GetOpPythonPath(const OperatorName &op_name) { - // almost all ops are defined in two main paths - const std::string ops_module = OP_PATH; - const std::string inner_ops_module = INNER_OP_PATH; - py::module mod = py::module::import(common::SafeCStr(ops_module)); - py::module inner_mod = py::module::import(common::SafeCStr(inner_ops_module)); - if (!py::hasattr(inner_mod, common::SafeCStr(op_name))) { - if (!py::hasattr(mod, common::SafeCStr(op_name))) { - MS_LOG(EXCEPTION) << ops_module << " or " << inner_ops_module << " don't have op:" << op_name; - } - return ops_module; - } - return inner_ops_module; -} - -ValuePtr CreatOpInstance(const OperatorAttrs &attrs, const OperatorName &op_name, const std::string &instance_name) { - std::string op_path = GetOpPythonPath(op_name); - py::module mod = py::module::import(common::SafeCStr(op_path)); - if (!py::hasattr(mod, common::SafeCStr(op_name))) { - MS_LOG(ERROR) << "Failure: op_path:" << op_path << " don't have attr " << op_name; - return nullptr; - } - std::vector arg_list; - (void)std::transform(attrs.begin(), attrs.end(), std::back_inserter(arg_list), - [](const Attr &attr) { return ValuePtrToPyData(attr.second); }); - py::object obj = - parse::python_adapter::CallPyFn(GET_OP_FUNCTION_PATH, GET_OP_FUNCTION, op_name, op_path, instance_name, arg_list); - ValuePtr op_instance = nullptr; - bool succ = parse::ConvertData(obj, &op_instance); - if (!succ) { - MS_LOG(ERROR) << "Failure:get Python op " << op_path << " from " << op_name << " fail"; - return nullptr; - } - return op_instance; -} - -AnfNodePtr ValuePtrToAnfNodePtr(const ValuePtr &value_ptr) { - auto value_node = NewValueNode(value_ptr); - MS_EXCEPTION_IF_NULL(value_node); - return value_node->cast(); -} - -static std::unordered_map int_tensor_map = {}; -AnfNodePtr CreateInt32Tensor(int32_t value) { - auto it = int_tensor_map.find(value); - if (it != int_tensor_map.end()) { - return it->second; - } - mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(py::int_(value), kInt32); - ValuePtr value_ptr = MakeValue(tensor_ptr); - auto anf_node_ptr = ValuePtrToAnfNodePtr(value_ptr); - int_tensor_map[value] = anf_node_ptr; - return anf_node_ptr; -} - -AnfNodePtr CreatTypeInt(int32_t value) { - ValuePtr value_ptr = MakeValue(std::make_shared(value)); - return ValuePtrToAnfNodePtr(value_ptr); -} - -AnfNodePtr CreatInt32Imm(int32_t value) { - ValuePtr value_ptr = MakeValue(std::make_shared(value)); - return ValuePtrToAnfNodePtr(value_ptr); -} - -std::string GetInstanceNameByCNode(const CNodePtr &cnode) { - PrimitivePtr prim = GetValueNode(cnode->input(0)); - if (!prim) { - MS_LOG(EXCEPTION) << "The first input of the cnode is not a PrimitivePtr."; - } - std::string instance_name = prim->instance_name(); - return HashInstanceName(instance_name); -} - -std::string HashInstanceName(const std::string &name) { - auto using_hash_name = common::GetEnv(USING_HASH_NAME); - std::string instance_name; - if ((using_hash_name.empty()) || (using_hash_name == "on")) { - instance_name = HashName(name); - } else { - instance_name = name; - } - return instance_name; -} - -Status GenerateGraph::Init(const CNodePtr &cnode) { - if (!cnode) { - MS_LOG(ERROR) << "Init:cnode is nullptr"; - return FAILED; - } - cnode_ = cnode; - func_graph_ = cnode->func_graph(); - if (!func_graph_) { - MS_LOG(ERROR) << "Init:func_graph_ is nullptr"; - return FAILED; - } - manager_ = func_graph_->manager(); - if (!manager_) { - MS_LOG(ERROR) << "Init:manager_ is nullptr"; - return FAILED; - } - scope_ = cnode_->scope(); - if (!scope_) { - MS_LOG(ERROR) << "Init:scope_ is nullptr"; - return FAILED; - } - virtual_input_node_ = std::make_shared(nullptr); - virtual_input_node_->set_scope(scope_); - instance_name_base_ = GetInstanceNameByCNode(cnode_); - name_idx_ = 0; - return SUCCESS; -} - -AnfNodePtr GenerateGraph::PushBack(const std::vector &inputs) { - CNodePtr cnode = func_graph_->NewCNode(inputs); // using NewCNode to creat anfnode - MS_EXCEPTION_IF_NULL(cnode); - cnode->set_scope(scope_); - if (inputs.size() < 2) { - MS_LOG(EXCEPTION) << "inputs.size() must be more than 1"; - } - (void)manager_->Replace(inputs.at(1), cnode); // using Replace function to insert cnode after inputs[0] - auto new_anf_node_ptr = cnode->cast(); - MS_EXCEPTION_IF_NULL(new_anf_node_ptr); - return new_anf_node_ptr; -} - -AnfNodePtr GenerateGraph::NewOpInst(const OperatorName &op_name, const OperatorAttrs &attrs) { - name_idx_++; - ValuePtr pyop_instance = CreatOpInstance(attrs, op_name, instance_name_base_ + op_name + std::to_string(name_idx_)); - if (pyop_instance == nullptr) { - MS_LOG(EXCEPTION) << "Failure:" << op_name << " CreatOpInstance failed"; - } - auto value_node = NewValueNode(pyop_instance); - return value_node->cast(); -} - -AnfNodePtr GenerateGraph::NewOpInst(const OperatorName &op_name) { - name_idx_++; - OperatorAttrs attrs; - ValuePtr pyop_instance = CreatOpInstance(attrs, op_name, instance_name_base_ + std::to_string(name_idx_)); - if (pyop_instance == nullptr) { - MS_LOG(EXCEPTION) << "Failure:" << op_name << " CreatOpInstance failed"; - } - auto value_node = NewValueNode(pyop_instance); - return value_node->cast(); -} -} // namespace parallel -} // namespace mindspore diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index b575718668..d664a59b19 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -326,7 +326,6 @@ __all__ = [ "ApplyCenteredRMSProp", "SpaceToBatchND", "BatchToSpaceND", - "ReverseSequence", "SquareSumAll", "BitwiseAnd", "BitwiseOr", @@ -341,12 +340,10 @@ __all__ = [ "ApproximateEqual", "InplaceUpdate", "InTopK", - "CropAndResize", "LRN", "Mod", "PopulationCount", "ParallelConcat", - "EmbeddingLookup", "Push", "Pull" ] From 47245e8ba0019778c897a93a73fb90f89f0f6106 Mon Sep 17 00:00:00 2001 From: "wangnan39@huawei.com" Date: Mon, 20 Jul 2020 08:51:37 +0800 Subject: [PATCH 04/13] rename operators of sparse optimizer --- .../cpu/sparse_apply_adam_cpu_kernel.h | 2 +- .../cpu/sparse_apply_ftrl_cpu_kernel.h | 14 +- .../cpu/sparse_apply_lazy_adam_cpu_kernel.h | 2 +- ...sparse_apply_proximal_adagrad_cpu_kernel.h | 15 +- mindspore/nn/optim/adam.py | 2 +- mindspore/nn/optim/ftrl.py | 3 +- mindspore/nn/optim/lazyadam.py | 2 +- mindspore/nn/optim/proximal_ada_grad.py | 3 +- mindspore/ops/operations/__init__.py | 9 +- mindspore/ops/operations/_inner_ops.py | 180 ----------------- mindspore/ops/operations/nn_ops.py | 186 +++++++++++++++++- tests/st/ops/cpu/test_sparse_apply_adam_op.py | 2 +- tests/st/ops/cpu/test_sparse_apply_ftrl_op.py | 2 +- .../test_sparse_apply_proximal_adagrad_op.py | 2 +- 14 files changed, 197 insertions(+), 227 deletions(-) diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_adam_cpu_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_adam_cpu_kernel.h index 5d3d4193f7..3a7a449246 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_adam_cpu_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_adam_cpu_kernel.h @@ -40,7 +40,7 @@ class SparseApplyAdamCPUKernel : public CPUKernel { bool use_nesterov_{false}; }; -MS_REG_CPU_KERNEL(SparseApplyAdam, +MS_REG_CPU_KERNEL(FusedSparseAdam, KernelAttr() .AddInputAttr(kNumberTypeFloat32) .AddInputAttr(kNumberTypeFloat32) diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_ftrl_cpu_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_ftrl_cpu_kernel.h index af8796d8a5..c24ce8c703 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_ftrl_cpu_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_ftrl_cpu_kernel.h @@ -42,19 +42,7 @@ class SparseApplyFtrlCPUKernel : public CPUKernel { float lr_power_{0}; }; -MS_REG_CPU_KERNEL(SparseApplyFtrl, - KernelAttr() - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeInt32) - .AddOutputAttr(kNumberTypeFloat32) - .AddOutputAttr(kNumberTypeFloat32) - .AddOutputAttr(kNumberTypeFloat32), - SparseApplyFtrlCPUKernel); - -MS_REG_CPU_KERNEL(SparseApplyFtrlNoReturn, +MS_REG_CPU_KERNEL(FusedSparseFtrl, KernelAttr() .AddInputAttr(kNumberTypeFloat32) .AddInputAttr(kNumberTypeFloat32) diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_lazy_adam_cpu_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_lazy_adam_cpu_kernel.h index ee95db8f33..e588702aea 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_lazy_adam_cpu_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_lazy_adam_cpu_kernel.h @@ -40,7 +40,7 @@ class SparseApplyLazyAdamCPUKernel : public CPUKernel { bool use_nesterov_{false}; }; -MS_REG_CPU_KERNEL(SparseApplyLazyAdam, +MS_REG_CPU_KERNEL(FusedSparseLazyAdam, KernelAttr() .AddInputAttr(kNumberTypeFloat32) .AddInputAttr(kNumberTypeFloat32) diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h index 56b180ec0b..616fb9b954 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h @@ -39,20 +39,7 @@ class SparseApplyProximalAdagradCPUKernel : public CPUKernel { size_t var_outer_dim_size_{1}; }; -MS_REG_CPU_KERNEL(SparseApplyProximalAdagrad, - KernelAttr() - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeFloat32) - .AddInputAttr(kNumberTypeInt32) - .AddOutputAttr(kNumberTypeFloat32) - .AddOutputAttr(kNumberTypeFloat32), - SparseApplyProximalAdagradCPUKernel); - -MS_REG_CPU_KERNEL(SparseApplyProximalAdagradNoReturn, +MS_REG_CPU_KERNEL(FusedSparseProximalAdagrad, KernelAttr() .AddInputAttr(kNumberTypeFloat32) .AddInputAttr(kNumberTypeFloat32) diff --git a/mindspore/nn/optim/adam.py b/mindspore/nn/optim/adam.py index eb6e64074f..794d2513e3 100755 --- a/mindspore/nn/optim/adam.py +++ b/mindspore/nn/optim/adam.py @@ -276,7 +276,7 @@ class Adam(Optimizer): self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) - self.sparse_opt = P.SparseApplyAdam(use_locking, use_nesterov) + self.sparse_opt = P.FusedSparseAdam(use_locking, use_nesterov) def construct(self, gradients): params = self.parameters diff --git a/mindspore/nn/optim/ftrl.py b/mindspore/nn/optim/ftrl.py index dd2ebddfa7..c0b11d6fa2 100644 --- a/mindspore/nn/optim/ftrl.py +++ b/mindspore/nn/optim/ftrl.py @@ -16,7 +16,6 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype -from mindspore.ops.operations import _inner_ops as inner from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from .optimizer import Optimizer, _apply_decay, _grad_scale @@ -138,7 +137,7 @@ class FTRL(Optimizer): self.decay_tf = tuple((lambda: True)() for x in self.parameters) self.hyper_map = C.HyperMap() self.opt = P.ApplyFtrl(use_locking=use_locking) - self.sparse_opt = inner.SparseApplyFtrlNoReturn(learning_rate, l1, l2, lr_power, use_locking=use_locking) + self.sparse_opt = P.FusedSparseFtrl(learning_rate, l1, l2, lr_power, use_locking=use_locking) def construct(self, grads): params = self.parameters diff --git a/mindspore/nn/optim/lazyadam.py b/mindspore/nn/optim/lazyadam.py index 7905398437..7df86bc277 100644 --- a/mindspore/nn/optim/lazyadam.py +++ b/mindspore/nn/optim/lazyadam.py @@ -182,7 +182,7 @@ class LazyAdam(Optimizer): self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) - self.sparse_opt = P.SparseApplyLazyAdam(use_locking, use_nesterov) + self.sparse_opt = P.FusedSparseLazyAdam(use_locking, use_nesterov) def construct(self, gradients): gradients = self.decay_weight(gradients) diff --git a/mindspore/nn/optim/proximal_ada_grad.py b/mindspore/nn/optim/proximal_ada_grad.py index 25cf438034..eba2b890df 100644 --- a/mindspore/nn/optim/proximal_ada_grad.py +++ b/mindspore/nn/optim/proximal_ada_grad.py @@ -16,7 +16,6 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype -from mindspore.ops.operations import _inner_ops as inner from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from .optimizer import Optimizer @@ -100,7 +99,7 @@ class ProximalAdagrad(Optimizer): self.weight_decay = weight_decay self.hyper_map = C.HyperMap() self.opt = P.ApplyProximalAdagrad(use_locking=use_locking) - self.sparse_opt = inner.SparseApplyProximalAdagradNoReturn(use_locking=use_locking) + self.sparse_opt = P.FusedSparseProximalAdagrad(use_locking=use_locking) def construct(self, grads): params = self.parameters diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index d664a59b19..9def0186c9 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -56,7 +56,7 @@ from .math_ops import (Abs, ACos, Asin, Asinh, AddN, AccumulateNV2, AssignAdd, A from .random_ops import (RandomChoiceWithMask, StandardNormal, Gamma, Poisson, UniformInt, UniformReal, RandomCategorical, Laplace) -from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, ApplyMomentum, BatchNorm, +from .nn_ops import (LSTM, SGD, Adam, FusedSparseAdam, FusedSparseLazyAdam, ApplyMomentum, BatchNorm, BiasAdd, Conv2D, DepthwiseConv2dNative, DropoutDoMask, DropoutGrad, Dropout, @@ -74,6 +74,7 @@ from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, Appl SparseSoftmaxCrossEntropyWithLogits, Tanh, TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl, SparseApplyFtrl, ApplyProximalAdagrad, SparseApplyProximalAdagrad, SparseApplyAdagradV2, SparseApplyFtrlV2, + FusedSparseFtrl, FusedSparseProximalAdagrad, ApplyAdaMax, ApplyAdadelta, ApplyAdagrad, ApplyAdagradV2, ApplyAddSign, ApplyPowerSign, ApplyGradientDescent, ApplyProximalGradientDescent, ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK) @@ -114,8 +115,8 @@ __all__ = [ 'MaxPool', 'TopK', 'Adam', - 'SparseApplyAdam', - 'SparseApplyLazyAdam', + 'FusedSparseAdam', + 'FusedSparseLazyAdam', 'Softplus', 'Softmax', 'Softsign', @@ -310,8 +311,10 @@ __all__ = [ "SpaceToBatch", "SparseApplyFtrl", "SparseApplyFtrlV2", + "FusedSparseFtrl", "ApplyProximalAdagrad", "SparseApplyProximalAdagrad", + "FusedSparseProximalAdagrad", "ApplyAdaMax", "ApplyAdadelta", "ApplyAdagrad", diff --git a/mindspore/ops/operations/_inner_ops.py b/mindspore/ops/operations/_inner_ops.py index 2d17da0028..e70e5b32d5 100644 --- a/mindspore/ops/operations/_inner_ops.py +++ b/mindspore/ops/operations/_inner_ops.py @@ -18,9 +18,6 @@ from ..._checkparam import Rel from ..._checkparam import Validator as validator from ...common import dtype as mstype -from ..._c_expression import signature_rw as sig_rw -from ..._c_expression import signature_kind as sig_kind -from ..._c_expression import signature_dtype as sig_dtype from ..primitive import PrimitiveWithInfer, prim_attr_register @@ -394,183 +391,6 @@ class AscendDequant(PrimitiveWithInfer): return mstype.float16 -class SparseApplyFtrlNoReturn(PrimitiveWithInfer): - """ - Update relevant entries according to the FTRL-proximal scheme. - - Args: - lr (float): The learning rate value, must be positive. - l1 (float): l1 regularization strength, must be greater than or equal to zero. - l2 (float): l2 regularization strength, must be greater than or equal to zero. - lr_power (float): Learning rate power controls how the learning rate decreases during training, - must be less than or equal to zero. Use fixed learning rate if `lr_power` is zero. - use_locking (bool): Use locks for update operation if True . Default: False. - - Inputs: - - **var** (Parameter): The variable to be updated. The data type must be float32. - - **accum** (Parameter): The accum to be updated, must be same type and shape as `var`. - - **linear** (Parameter): The linear to be updated, must be same type and shape as `var`. - - **grad** (Tensor): A tensor of the same type as `var`, for the gradient. - - **indices** (Tensor): A vector of indices into the first dimension of `var` and `accum`. The shape - of `indices` must be the same as `grad` in first dimension. The type must be int32. - - Outputs: - Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. - - - **var** (Tensor) - A Tensor with shape (1,). - - **accum** (Tensor) - A Tensor with shape (1,). - - **linear** (Tensor) - A Tensor with shape (1,). - - Examples: - >>> import mindspore - >>> import mindspore.nn as nn - >>> import numpy as np - >>> from mindspore import Parameter - >>> from mindspore import Tensor - >>> from mindspore.ops import operations as P - >>> class SparseApplyFtrlNet(nn.Cell): - >>> def __init__(self): - >>> super(SparseApplyFtrlNet, self).__init__() - >>> self.sparse_apply_ftrl = P.SparseApplyFtrlV2(lr=0.01, l1=0.0, l2=0.0, lr_power=-0.5) - >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") - >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") - >>> self.linear = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="linear") - >>> - >>> def construct(self, grad, indices): - >>> out = self.sparse_apply_ftrl(self.var, self.accum, self.linear, grad, indices) - >>> return out - >>> - >>> net = SparseApplyFtrlNet() - >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) - >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) - >>> output = net(grad, indices) - """ - __mindspore_signature__ = ( - ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) - ) - - @prim_attr_register - def __init__(self, lr, l1, l2, lr_power, use_locking=False): - self.init_prim_io_names(inputs=['var', 'accum', 'linear', 'grad', 'indices'], - outputs=['output']) - validator.check_value_type("lr", lr, [float], self.name) - validator.check_value_type("l1", l1, [float], self.name) - validator.check_value_type("l2", l2, [float], self.name) - validator.check_value_type("lr_power", lr_power, [float], self.name) - self.lr = validator.check_number_range("lr", lr, 0.0, float("inf"), Rel.INC_NEITHER, self.name) - self.l1 = validator.check_number_range("l1", l1, 0.0, float("inf"), Rel.INC_LEFT, self.name) - self.l2 = validator.check_number_range("l2", l2, 0.0, float("inf"), Rel.INC_LEFT, self.name) - self.lr_power = validator.check_number("lr_power", lr_power, 0, Rel.LE, self.name) - self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) - self.add_prim_attr('primitive_target', 'CPU') - - def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, indices_shape): - validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) - validator.check('var shape', var_shape, 'linear shape', linear_shape, Rel.EQ, self.name) - if len(var_shape) > 1: - validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) - validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) - validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) - return [1], [1], [1] - - def infer_dtype(self, var_dtype, accum_dtype, linear_dtype, grad_dtype, indices_dtype): - args = {"var_dtype": var_dtype, "accum_dtype": accum_dtype, - "linear_dtype": linear_dtype, "grad_dtype": grad_dtype} - validator.check_tensor_type_same(args, [mstype.float32], self.name) - validator.check_tensor_type_same({"indices_dtype": indices_dtype}, [mstype.int32], self.name) - return var_dtype, accum_dtype, linear_dtype - - -class SparseApplyProximalAdagradNoReturn(PrimitiveWithInfer): - r""" - Updates relevant entries according to the proximal adagrad algorithm. - - .. math:: - accum += grad * grad - .. math:: - \text{prox_v} = var - lr * grad * \frac{1}{\sqrt{accum}} - .. math:: - var = \frac{sign(\text{prox_v})}{1 + lr * l2} * \max(\left| \text{prox_v} \right| - lr * l1, 0) - - Args: - use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. - - Inputs: - - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. - - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. - - **lr** (Tensor): The learning rate value. The data type must be float32. - - **l1** (Tensor): l1 regularization strength. The data type must be float32. - - **l2** (Tensor): l2 regularization strength. The data type must be float32. - - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. - - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. The data type - must be int32. - - Outputs: - Tuple of 2 Tensor, this operator will update the input parameters directly, the outputs are useless. - - - **var** (Tensor) - A Tensor with shape (1,). - - **accum** (Tensor) - A Tensor with shape (1,). - - Examples: - >>> import numpy as np - >>> import mindspore.nn as nn - >>> from mindspore import Tensor, Parameter - >>> from mindspore.ops import operations as P - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.sparse_apply_proximal_adagrad = P.SparseApplyProximalAdagradV2() - >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") - >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") - >>> self.lr = Tensor(0.01, mstype.float32) - >>> self.l1 = Tensor(0.0, mstype.float32) - >>> self.l2 = Tensor(0.0, mstype.float32) - >>> def construct(self, grad, indices): - >>> out = self.sparse_apply_proximal_adagrad(self.var, self.accum, self.lr, self.l1, - >>> self.l2, grad, indices) - >>> return out - >>> net = Net() - >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) - >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) - >>> output = net(grad, indices) - """ - __mindspore_signature__ = ( - ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) - ) - - @prim_attr_register - def __init__(self, use_locking=False): - self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad', 'indices'], - outputs=['output']) - self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) - self.add_prim_attr('primitive_target', 'CPU') - - def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape, indices_shape): - validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) - return [1], [1] - - def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype, indices_dtype): - args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} - validator.check_tensor_type_same(args, [mstype.float32], self.name) - validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, [mstype.float32], self.name) - validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, [mstype.float32], self.name) - validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, [mstype.float32], self.name) - valid_types = [mstype.int16, mstype.int32, mstype.int64, - mstype.uint16, mstype.uint32, mstype.uint64] - validator.check_tensor_type_same({'indices': indices_dtype}, valid_types, self.name) - return var_dtype, accum_dtype - - class LinSpace(PrimitiveWithInfer): r""" Generates values in an interval. And return the corresponding interpolation accroding to assist. diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 83e6454a00..1c8b24a112 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -2912,7 +2912,7 @@ class Adam(PrimitiveWithInfer): return var_dtype, m_dtype, v_dtype -class SparseApplyAdam(PrimitiveWithInfer): +class FusedSparseAdam(PrimitiveWithInfer): r""" Merge the duplicate value of the gradient and then updates parameters by Adaptive Moment Estimation (Adam) algorithm. This operator is used when the gradient is sparse. @@ -2974,7 +2974,7 @@ class SparseApplyAdam(PrimitiveWithInfer): >>> class Net(nn.Cell): >>> def __init__(self): >>> super(Net, self).__init__() - >>> self.sparse_apply_adam = P.SparseApplyAdam() + >>> self.sparse_apply_adam = P.FusedSparseAdam() >>> self.var = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="var") >>> self.m = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="m") >>> self.v = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="v") @@ -3020,7 +3020,6 @@ class SparseApplyAdam(PrimitiveWithInfer): self.init_prim_io_names(inputs=['var', 'm', 'v', 'beta1_power', 'beta2_power', 'lr', 'beta1', 'beta2', 'epsilon', 'grad', 'indices'], outputs=['var', 'm', 'v']) - self.add_prim_attr('primitive_target', 'CPU') def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape, indices_shape): @@ -3046,7 +3045,7 @@ class SparseApplyAdam(PrimitiveWithInfer): return var_dtype, m_dtype, v_dtype -class SparseApplyLazyAdam(PrimitiveWithInfer): +class FusedSparseLazyAdam(PrimitiveWithInfer): r""" Merge the duplicate value of the gradient and then updates parameters by Adaptive Moment Estimation (Adam) algorithm. This operator is used when the gradient is sparse. The behavior is not equivalent to the @@ -3109,7 +3108,7 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): >>> class Net(nn.Cell): >>> def __init__(self): >>> super(Net, self).__init__() - >>> self.sparse_apply_lazyadam = P.SparseApplyLazyAdam() + >>> self.sparse_apply_lazyadam = P.FusedSparseLazyAdam() >>> self.var = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="var") >>> self.m = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="m") >>> self.v = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="v") @@ -3155,7 +3154,6 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): self.init_prim_io_names(inputs=['var', 'm', 'v', 'beta1_power', 'beta2_power', 'lr', 'beta1', 'beta2', 'epsilon', 'grad', 'indices'], outputs=['var', 'm', 'v']) - self.add_prim_attr('primitive_target', 'CPU') def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape, indices_shape): @@ -3182,6 +3180,182 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): return var_dtype, m_dtype, v_dtype +class FusedSparseFtrl(PrimitiveWithInfer): + """ + Merge the duplicate value of the gradient and then update relevant entries according to the FTRL-proximal scheme. + + Args: + lr (float): The learning rate value, must be positive. + l1 (float): l1 regularization strength, must be greater than or equal to zero. + l2 (float): l2 regularization strength, must be greater than or equal to zero. + lr_power (float): Learning rate power controls how the learning rate decreases during training, + must be less than or equal to zero. Use fixed learning rate if `lr_power` is zero. + use_locking (bool): Use locks for update operation if True . Default: False. + + Inputs: + - **var** (Parameter): The variable to be updated. The data type must be float32. + - **accum** (Parameter): The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter): The linear to be updated, must be same type and shape as `var`. + - **grad** (Tensor): A tensor of the same type as `var`, for the gradient. + - **indices** (Tensor): A vector of indices into the first dimension of `var` and `accum`. The shape + of `indices` must be the same as `grad` in first dimension. The type must be int32. + + Outputs: + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. + + - **var** (Tensor) - A Tensor with shape (1,). + - **accum** (Tensor) - A Tensor with shape (1,). + - **linear** (Tensor) - A Tensor with shape (1,). + + Examples: + >>> import mindspore + >>> import mindspore.nn as nn + >>> import numpy as np + >>> from mindspore import Parameter + >>> from mindspore import Tensor + >>> from mindspore.ops import operations as P + >>> class SparseApplyFtrlNet(nn.Cell): + >>> def __init__(self): + >>> super(SparseApplyFtrlNet, self).__init__() + >>> self.sparse_apply_ftrl = P.FusedSparseFtrl(lr=0.01, l1=0.0, l2=0.0, lr_power=-0.5) + >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") + >>> self.linear = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="linear") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_ftrl(self.var, self.accum, self.linear, grad, indices) + >>> return out + >>> + >>> net = SparseApplyFtrlNet() + >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) + >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) + >>> output = net(grad, indices) + """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, l1, l2, lr_power, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'linear', 'grad', 'indices'], + outputs=['output']) + validator.check_value_type("lr", lr, [float], self.name) + validator.check_value_type("l1", l1, [float], self.name) + validator.check_value_type("l2", l2, [float], self.name) + validator.check_value_type("lr_power", lr_power, [float], self.name) + self.lr = validator.check_number_range("lr", lr, 0.0, float("inf"), Rel.INC_NEITHER, self.name) + self.l1 = validator.check_number_range("l1", l1, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.l2 = validator.check_number_range("l2", l2, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.lr_power = validator.check_number("lr_power", lr_power, 0, Rel.LE, self.name) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + + def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('var shape', var_shape, 'linear shape', linear_shape, Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return [1], [1], [1] + + def infer_dtype(self, var_dtype, accum_dtype, linear_dtype, grad_dtype, indices_dtype): + args = {"var_dtype": var_dtype, "accum_dtype": accum_dtype, + "linear_dtype": linear_dtype, "grad_dtype": grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({"indices_dtype": indices_dtype}, [mstype.int32], self.name) + return var_dtype, accum_dtype, linear_dtype + + +class FusedSparseProximalAdagrad(PrimitiveWithInfer): + r""" + Merge the duplicate value of the gradient and then Updates relevant entries according to the proximal adagrad + algorithm. + + .. math:: + accum += grad * grad + .. math:: + \text{prox_v} = var - lr * grad * \frac{1}{\sqrt{accum}} + .. math:: + var = \frac{sign(\text{prox_v})}{1 + lr * l2} * \max(\left| \text{prox_v} \right| - lr * l1, 0) + + Args: + use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. + - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Tensor): The learning rate value. The data type must be float32. + - **l1** (Tensor): l1 regularization strength. The data type must be float32. + - **l2** (Tensor): l2 regularization strength. The data type must be float32. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. The data type + must be int32. + + Outputs: + Tuple of 2 Tensor, this operator will update the input parameters directly, the outputs are useless. + + - **var** (Tensor) - A Tensor with shape (1,). + - **accum** (Tensor) - A Tensor with shape (1,). + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.sparse_apply_proximal_adagrad = P.FusedSparseProximalAdagrad() + >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") + >>> self.lr = Tensor(0.01, mstype.float32) + >>> self.l1 = Tensor(0.0, mstype.float32) + >>> self.l2 = Tensor(0.0, mstype.float32) + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_proximal_adagrad(self.var, self.accum, self.lr, self.l1, + >>> self.l2, grad, indices) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) + >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) + >>> output = net(grad, indices) + """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad', 'indices'], + outputs=['output']) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + + def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape, indices_shape): + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + return [1], [1] + + def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype, indices_dtype): + args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, [mstype.float32], self.name) + valid_types = [mstype.int16, mstype.int32, mstype.int64, + mstype.uint16, mstype.uint32, mstype.uint64] + validator.check_tensor_type_same({'indices': indices_dtype}, valid_types, self.name) + return var_dtype, accum_dtype + + class BinaryCrossEntropy(PrimitiveWithInfer): r""" Computes the Binary Cross Entropy between the target and the output. diff --git a/tests/st/ops/cpu/test_sparse_apply_adam_op.py b/tests/st/ops/cpu/test_sparse_apply_adam_op.py index 06b4a70b39..887f76313c 100644 --- a/tests/st/ops/cpu/test_sparse_apply_adam_op.py +++ b/tests/st/ops/cpu/test_sparse_apply_adam_op.py @@ -32,7 +32,7 @@ epsilon = 1e-8 class Net(nn.Cell): def __init__(self): super(Net, self).__init__() - self.sparse_apply_adam = P.SparseApplyAdam() + self.sparse_apply_adam = P.FusedSparseAdam() self.var = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="var") self.m = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="m") self.v = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="v") diff --git a/tests/st/ops/cpu/test_sparse_apply_ftrl_op.py b/tests/st/ops/cpu/test_sparse_apply_ftrl_op.py index babaefbd86..3071c54f04 100644 --- a/tests/st/ops/cpu/test_sparse_apply_ftrl_op.py +++ b/tests/st/ops/cpu/test_sparse_apply_ftrl_op.py @@ -25,7 +25,7 @@ import mindspore.common.dtype as mstype class Net(nn.Cell): def __init__(self): super(Net, self).__init__() - self.sparse_apply_ftrl = P.SparseApplyFtrl(lr=0.001, l1=0.0, l2=0.0, lr_power=-0.5) + self.sparse_apply_ftrl = P.FusedSparseFtrl(lr=0.001, l1=0.0, l2=0.0, lr_power=-0.5) self.var = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="var") self.accum = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="accum") self.linear = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="linear") diff --git a/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py b/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py index c2a129a86c..67510a73be 100644 --- a/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py +++ b/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py @@ -25,7 +25,7 @@ import mindspore.common.dtype as mstype class Net(nn.Cell): def __init__(self): super(Net, self).__init__() - self.sparse_apply_proximal_adagrad = P.SparseApplyProximalAdagrad() + self.sparse_apply_proximal_adagrad = P.FusedSparseProximalAdagrad() self.var = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="var") self.accum = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="accum") self.lr = 0.01 From a536b68c84e56918f9d5abae70b4973edebd1f88 Mon Sep 17 00:00:00 2001 From: yanzhenxiang2020 Date: Sat, 11 Jul 2020 17:56:17 +0800 Subject: [PATCH 05/13] add FusedSparse(Adam/LazyAdam/Ftrl/ProximalAdagrad) for aicpu --- mindspore/ops/_op_impl/aicpu/__init__.py | 4 ++ .../ops/_op_impl/aicpu/fused_sparse_adam.py | 46 ++++++++++++++++ .../ops/_op_impl/aicpu/fused_sparse_ftrl.py | 41 ++++++++++++++ .../_op_impl/aicpu/fused_sparse_lazy_adam.py | 46 ++++++++++++++++ .../aicpu/fused_sparse_proximal_adagrad.py | 39 ++++++++++++++ .../test_aicpu_ops/test_fused_sparse_adam.py | 53 +++++++++++++++++++ .../test_aicpu_ops/test_fused_sparse_ftrl.py | 50 +++++++++++++++++ .../test_fused_sparse_lazy_adam.py | 53 +++++++++++++++++++ .../test_fused_sparse_proximal_adagrad.py | 47 ++++++++++++++++ 9 files changed, 379 insertions(+) create mode 100644 mindspore/ops/_op_impl/aicpu/fused_sparse_adam.py create mode 100644 mindspore/ops/_op_impl/aicpu/fused_sparse_ftrl.py create mode 100644 mindspore/ops/_op_impl/aicpu/fused_sparse_lazy_adam.py create mode 100644 mindspore/ops/_op_impl/aicpu/fused_sparse_proximal_adagrad.py create mode 100644 tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_adam.py create mode 100644 tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_ftrl.py create mode 100644 tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_lazy_adam.py create mode 100644 tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_proximal_adagrad.py diff --git a/mindspore/ops/_op_impl/aicpu/__init__.py b/mindspore/ops/_op_impl/aicpu/__init__.py index 5551dc58b4..b321db47e0 100644 --- a/mindspore/ops/_op_impl/aicpu/__init__.py +++ b/mindspore/ops/_op_impl/aicpu/__init__.py @@ -44,3 +44,7 @@ from .laplace import _laplace_aicpu from .strided_slice import _strided_slice_aicpu from .strided_slice_grad import _strided_slice_grad_aicpu from .end_of_sequence import _end_of_sequence_aicpu +from .fused_sparse_adam import _fused_sparse_adam_aicpu +from .fused_sparse_lazy_adam import _fused_sparse_lazy_adam_aicpu +from .fused_sparse_ftrl import _fused_sparse_ftrl_aicpu +from .fused_sparse_proximal_adagrad import _fused_sparse_proximal_adagrad_aicpu diff --git a/mindspore/ops/_op_impl/aicpu/fused_sparse_adam.py b/mindspore/ops/_op_impl/aicpu/fused_sparse_adam.py new file mode 100644 index 0000000000..ef56ef7427 --- /dev/null +++ b/mindspore/ops/_op_impl/aicpu/fused_sparse_adam.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedSparseAdam op""" +from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType + +fused_sparse_adam_op_info = AiCPURegOp("FusedSparseAdam") \ + .fusion_type("OPAQUE") \ + .attr("use_locking", "bool") \ + .attr("use_nesterov", "bool") \ + .input(0, "var", "required") \ + .input(1, "m", "required") \ + .input(2, "v", "required") \ + .input(3, "beta1_power", "required") \ + .input(4, "beta2_power", "required") \ + .input(5, "lr", "required") \ + .input(6, "beta1", "required") \ + .input(7, "beta2", "required") \ + .input(8, "epsilon", "required") \ + .input(9, "grad", "required") \ + .input(10, "indices", "required") \ + .output(0, "var", "required") \ + .output(1, "m", "required") \ + .output(2, "v", "required") \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + +@op_info_register(fused_sparse_adam_op_info) +def _fused_sparse_adam_aicpu(): + """FusedSparseAdam aicpu register""" + return diff --git a/mindspore/ops/_op_impl/aicpu/fused_sparse_ftrl.py b/mindspore/ops/_op_impl/aicpu/fused_sparse_ftrl.py new file mode 100644 index 0000000000..719ac90620 --- /dev/null +++ b/mindspore/ops/_op_impl/aicpu/fused_sparse_ftrl.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedSparseFtrl op""" +from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType + +fused_sparse_ftrl_op_info = AiCPURegOp("FusedSparseFtrl") \ + .fusion_type("OPAQUE") \ + .attr("lr", "float") \ + .attr("l1", "float") \ + .attr("l2", "float") \ + .attr("lr_power", "float") \ + .attr("use_locking", "bool") \ + .input(0, "var", "required") \ + .input(1, "accum", "required") \ + .input(2, "linear", "required") \ + .input(3, "grad", "required") \ + .input(4, "indices", "required") \ + .output(0, "var", "required") \ + .output(1, "accum", "required") \ + .output(2, "linear", "required") \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.I32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + +@op_info_register(fused_sparse_ftrl_op_info) +def _fused_sparse_ftrl_aicpu(): + """FusedSparseFtrl aicpu register""" + return diff --git a/mindspore/ops/_op_impl/aicpu/fused_sparse_lazy_adam.py b/mindspore/ops/_op_impl/aicpu/fused_sparse_lazy_adam.py new file mode 100644 index 0000000000..708ec7f77c --- /dev/null +++ b/mindspore/ops/_op_impl/aicpu/fused_sparse_lazy_adam.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedSparseLazyAdam op""" +from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType + +fused_sparse_lazy_adam_op_info = AiCPURegOp("FusedSparseLazyAdam") \ + .fusion_type("OPAQUE") \ + .attr("use_locking", "bool") \ + .attr("use_nesterov", "bool") \ + .input(0, "var", "required") \ + .input(1, "m", "required") \ + .input(2, "v", "required") \ + .input(3, "beta1_power", "required") \ + .input(4, "beta2_power", "required") \ + .input(5, "lr", "required") \ + .input(6, "beta1", "required") \ + .input(7, "beta2", "required") \ + .input(8, "epsilon", "required") \ + .input(9, "grad", "required") \ + .input(10, "indices", "required") \ + .output(0, "var", "required") \ + .output(1, "m", "required") \ + .output(2, "v", "required") \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + +@op_info_register(fused_sparse_lazy_adam_op_info) +def _fused_sparse_lazy_adam_aicpu(): + """FusedSparseLazyAdam aicpu register""" + return diff --git a/mindspore/ops/_op_impl/aicpu/fused_sparse_proximal_adagrad.py b/mindspore/ops/_op_impl/aicpu/fused_sparse_proximal_adagrad.py new file mode 100644 index 0000000000..c64e17c99f --- /dev/null +++ b/mindspore/ops/_op_impl/aicpu/fused_sparse_proximal_adagrad.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedSparseProximalAdagrad op""" +from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType + +fused_sparse_proximal_adagrad_op_info = AiCPURegOp("FusedSparseProximalAdagrad") \ + .fusion_type("OPAQUE") \ + .attr("use_locking", "bool") \ + .input(0, "var", "required") \ + .input(1, "accum", "required") \ + .input(2, "lr", "required") \ + .input(3, "l1", "required") \ + .input(4, "l2", "required") \ + .input(5, "grad", "required") \ + .input(6, "indices", "required") \ + .output(0, "var", "required") \ + .output(1, "accum", "required") \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, + DataType.F32_Default) \ + .get_op_info() + +@op_info_register(fused_sparse_proximal_adagrad_op_info) +def _fused_sparse_proximal_adagrad_aicpu(): + """FusedSparseProximalAdagrad aicpu register""" + return diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_adam.py b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_adam.py new file mode 100644 index 0000000000..1edfd088a5 --- /dev/null +++ b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_adam.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +import mindspore.context as context +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +beta1_power = 0.9 +beta2_power = 0.999 +lr = 0.001 +beta1 = 0.9 +beta2 = 0.999 +epsilon = 1e-8 + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.fused_sparse_adam = P.FusedSparseAdam() + self.var = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="var") + self.m = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="m") + self.v = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="v") + + def construct(self, grad, indices): + return self.fused_sparse_adam(self.var, self.m, self.v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, + grad, indices) + +def test_net(): + gradient = Tensor(np.array([0.22948648, 0.14569908, 0.92861906, 0.66870148]) + .reshape([2, 1, 2]).astype(np.float32)) + indices = Tensor([0, 1], mstype.int32) + net = Net() + output = net(gradient, indices) + print(output) + print(net.var.default_input) + print(net.m.default_input) + print(net.v.default_input) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_ftrl.py b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_ftrl.py new file mode 100644 index 0000000000..6b35406c6f --- /dev/null +++ b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_ftrl.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.common.dtype as mstype +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +lr = 0.01 +l1 = 0.0 +l2 = 0.0 +lr_power = -0.5 + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.fused_sparse_ftrl = P.FusedSparseFtrl(lr=0.1, l1=0.0, l2=0.0, lr_power=-0.5) + self.var = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="var") + self.accum = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="accum") + self.linear = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="linear") + + def construct(self, grad, indices): + return self.fused_sparse_ftrl(self.var, self.accum, self.linear, grad, indices) + +def test_net(): + gradient = Tensor(np.array([-3, 2, 3, 0, 0, 0, -4, -1, -2]) + .reshape([3, 3]).astype(np.float32)) + indices = Tensor(np.ones([3]), mstype.int32) + net = Net() + output = net(gradient, indices) + print(output) + print(net.var.default_input) + print(net.accum.default_input) + print(net.linear.default_input) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_lazy_adam.py b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_lazy_adam.py new file mode 100644 index 0000000000..43b28710a1 --- /dev/null +++ b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_lazy_adam.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.common.dtype as mstype +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +beta1_power = 0.9 +beta2_power = 0.999 +lr = 0.001 +beta1 = 0.9 +beta2 = 0.999 +epsilon = 1e-8 + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.fused_sparse_lazy_adam = P.FusedSparseLazyAdam() + self.var = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="var") + self.m = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="m") + self.v = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="v") + + def construct(self, grad, indices): + return self.fused_sparse_lazy_adam(self.var, self.m, self.v, beta1_power, beta2_power, + lr, beta1, beta2, epsilon, grad, indices) + +def test_net(): + gradient = Tensor(np.array([0.22948648, 0.14569908, 0.92861906, 0.66870148]) + .reshape([2, 1, 2]).astype(np.float32)) + indices = Tensor([0, 1], mstype.int32) + net = Net() + output = net(gradient, indices) + print(output) + print(net.var.default_input) + print(net.m.default_input) + print(net.v.default_input) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_proximal_adagrad.py b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_proximal_adagrad.py new file mode 100644 index 0000000000..b5d837a2ed --- /dev/null +++ b/tests/st/ops/ascend/test_aicpu_ops/test_fused_sparse_proximal_adagrad.py @@ -0,0 +1,47 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +import mindspore.common.dtype as mstype +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.fused_sparse_proximal_adagrad = P.FusedSparseProximalAdagrad() + self.var = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="var") + self.accum = Parameter(Tensor(np.ones([3, 3]).astype(np.float32)), name="accum") + self.lr = 0.01 + self.l1 = 0.0 + self.l2 = 0.0 + + def construct(self, grad, indices): + return self.fused_sparse_proximal_adagrad(self.var, self.accum, self.lr, self.l1, self.l2, + grad, indices) + +def test_net(): + gradient = Tensor(np.array([-3, 2, 3, 0, 0, 0, -4, -1, -2]) + .reshape([3, 3]).astype(np.float32)) + indices = Tensor(np.ones([3]), mstype.int32) + net = Net() + output = net(gradient, indices) + print(output) + print(net.var.default_input) + print(net.accum.default_input) From 000e9915447ecd1492afea07dfa2f91781a7bd0d Mon Sep 17 00:00:00 2001 From: fangzehua Date: Tue, 28 Jul 2020 11:18:39 +0800 Subject: [PATCH 06/13] remove incubator Multilply declare --- mindspore/ccsrc/transform/graph_ir/op_declare.cc | 11 ----------- mindspore/ccsrc/transform/graph_ir/op_declare.h | 5 ----- mindspore/ccsrc/transform/op_declare.cc | 11 ----------- 3 files changed, 27 deletions(-) diff --git a/mindspore/ccsrc/transform/graph_ir/op_declare.cc b/mindspore/ccsrc/transform/graph_ir/op_declare.cc index b42519900e..a730093606 100644 --- a/mindspore/ccsrc/transform/graph_ir/op_declare.cc +++ b/mindspore/ccsrc/transform/graph_ir/op_declare.cc @@ -309,12 +309,6 @@ INPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{1, INPUT_DESC(features)}, {2, INPUT ATTR_MAP(SoftmaxCrossEntropyWithLogits) = EMPTY_ATTR_MAP; OUTPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(backprop)}}; -// MeanGrad -INPUT_MAP(MeanGrad) = {{1, INPUT_DESC(x)}}; -INPUT_ATTR_MAP(MeanGrad) = {{2, ATTR_DESC(mean_grad_output_shape_value, kOpFormat_NHWC, - AnyTraits>(), AnyTraits())}}; -ATTR_MAP(MeanGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; - INPUT_MAP(SliceD) = {{1, INPUT_DESC(x)}}; INPUT_ATTR_MAP(SliceD) = {{2, ATTR_DESC(offsets, AnyTraits(), AnyTraits>())}, {3, ATTR_DESC(size, AnyTraits(), AnyTraits>())}}; @@ -431,11 +425,6 @@ INPUT_MAP(TopK) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(k)}}; ATTR_MAP(TopK) = {{"sorted", ATTR_DESC(sorted, AnyTraits())}}; OUTPUT_MAP(TopK) = {{0, OUTPUT_DESC(values)}, {1, OUTPUT_DESC(indices)}}; -// Multiply -INPUT_MAP(Multiply) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}}; -ATTR_MAP(Multiply) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Multiply) = {{0, OUTPUT_DESC(z)}}; - // TileD INPUT_MAP(TileD) = {{1, INPUT_DESC(x)}}; INPUT_ATTR_MAP(TileD) = {{2, ATTR_DESC(multiples, AnyTraits(), AnyTraits>())}}; diff --git a/mindspore/ccsrc/transform/graph_ir/op_declare.h b/mindspore/ccsrc/transform/graph_ir/op_declare.h index b849461d56..472dd00275 100755 --- a/mindspore/ccsrc/transform/graph_ir/op_declare.h +++ b/mindspore/ccsrc/transform/graph_ir/op_declare.h @@ -70,8 +70,6 @@ DECLARE_OP_ADAPTER(AssignSub) DECLARE_OP_USE_OUTPUT(AssignSub) DECLARE_OP_ADAPTER(ReduceMean) -DECLARE_OP_ADAPTER(Multiply) -DECLARE_OP_USE_OUTPUT(Multiply) // ** Distributed Operations ** DECLARE_OP_ADAPTER(HcomReduceScatter) @@ -327,9 +325,6 @@ DECLARE_OP_USE_OUTPUT(MatMulV2) DECLARE_OP_ADAPTER(SoftmaxCrossEntropyWithLogits) DECLARE_OP_USE_OUTPUT(SoftmaxCrossEntropyWithLogits) -DECLARE_OP_ADAPTER(MeanGrad) -DECLARE_OP_USE_INPUT_ATTR(MeanGrad) - DECLARE_OP_ADAPTER(Assign) DECLARE_OP_USE_OUTPUT(Assign) DECLARE_OP_ADAPTER(Constant) diff --git a/mindspore/ccsrc/transform/op_declare.cc b/mindspore/ccsrc/transform/op_declare.cc index 9d86b30b17..fc0ac095ab 100644 --- a/mindspore/ccsrc/transform/op_declare.cc +++ b/mindspore/ccsrc/transform/op_declare.cc @@ -293,12 +293,6 @@ INPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{1, INPUT_DESC(features)}, {2, INPUT ATTR_MAP(SoftmaxCrossEntropyWithLogits) = EMPTY_ATTR_MAP; OUTPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(backprop)}}; -// MeanGrad -INPUT_MAP(MeanGrad) = {{1, INPUT_DESC(x)}}; -INPUT_ATTR_MAP(MeanGrad) = {{2, ATTR_DESC(mean_grad_output_shape_value, kOpFormat_NHWC, - AnyTraits>(), AnyTraits())}}; -ATTR_MAP(MeanGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; - INPUT_MAP(SliceD) = {{1, INPUT_DESC(x)}}; INPUT_ATTR_MAP(SliceD) = {{2, ATTR_DESC(offsets, AnyTraits(), AnyTraits>())}, {3, ATTR_DESC(size, AnyTraits(), AnyTraits>())}}; @@ -415,11 +409,6 @@ INPUT_MAP(TopK) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(k)}}; ATTR_MAP(TopK) = {{"sorted", ATTR_DESC(sorted, AnyTraits())}}; OUTPUT_MAP(TopK) = {{0, OUTPUT_DESC(values)}, {1, OUTPUT_DESC(indices)}}; -// Multiply -INPUT_MAP(Multiply) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}}; -ATTR_MAP(Multiply) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Multiply) = {{0, OUTPUT_DESC(z)}}; - // TileD INPUT_MAP(TileD) = {{1, INPUT_DESC(x)}}; INPUT_ATTR_MAP(TileD) = {{2, ATTR_DESC(multiples, AnyTraits(), AnyTraits>())}}; From f5e899abd682eb6206e5011614e4c60f8a082d7c Mon Sep 17 00:00:00 2001 From: hanjun996 Date: Tue, 28 Jul 2020 19:23:33 +0800 Subject: [PATCH 07/13] sync tdt --- mindspore/ccsrc/utils/context/ms_context.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index d6381ec7e8..4a9be65869 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -192,18 +192,22 @@ bool MsContext::OpenTsd() { } MS_LOG(INFO) << "Device id = " << device_id << ", rank size = " << rank_size << "."; +#ifdef ENABLE_TDTQUE int32_t initStatus = tdt::TdtHostInit(device_id); if (initStatus != TDT_OK_CODE) { MS_LOG(EXCEPTION) << "Init tsd failed, status = " << initStatus << "."; return false; } - tdt_print_ = std::thread(TensorPrint()); +#endif TDT_StatusT status = tdt::TsdClient::GetInstance()->Open(device_id, rank_size); if (status != TDT_OK) { MS_LOG(EXCEPTION) << "Device " << device_id << " is occupied, open tsd failed, status = " << status << "."; return false; } tsd_ref_++; +#ifdef ENABLE_TDTQUE + tdt_print_ = std::thread(TensorPrint()); +#endif MS_LOG(INFO) << "Open and init tsd successful, tsd reference = " << tsd_ref_ << "."; return true; } From d52270029b1220e043dcf7c5172cf31dbc056cfa Mon Sep 17 00:00:00 2001 From: hanjun996 Date: Wed, 29 Jul 2020 19:10:06 +0800 Subject: [PATCH 08/13] modify tdt --- mindspore/ccsrc/utils/context/ms_context.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index 4a9be65869..720c77e23e 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -192,13 +192,6 @@ bool MsContext::OpenTsd() { } MS_LOG(INFO) << "Device id = " << device_id << ", rank size = " << rank_size << "."; -#ifdef ENABLE_TDTQUE - int32_t initStatus = tdt::TdtHostInit(device_id); - if (initStatus != TDT_OK_CODE) { - MS_LOG(EXCEPTION) << "Init tsd failed, status = " << initStatus << "."; - return false; - } -#endif TDT_StatusT status = tdt::TsdClient::GetInstance()->Open(device_id, rank_size); if (status != TDT_OK) { MS_LOG(EXCEPTION) << "Device " << device_id << " is occupied, open tsd failed, status = " << status << "."; @@ -206,6 +199,11 @@ bool MsContext::OpenTsd() { } tsd_ref_++; #ifdef ENABLE_TDTQUE + int32_t initStatus = tdt::TdtHostInit(device_id); + if (initStatus != TDT_OK_CODE) { + MS_LOG(EXCEPTION) << "Init tsd failed, status = " << initStatus << "."; + return false; + } tdt_print_ = std::thread(TensorPrint()); #endif MS_LOG(INFO) << "Open and init tsd successful, tsd reference = " << tsd_ref_ << "."; From 374772035a6ad52f05b38ff5165c765bf6b32c47 Mon Sep 17 00:00:00 2001 From: peixu_ren Date: Wed, 29 Jul 2020 17:28:53 -0400 Subject: [PATCH 09/13] Refactor random uniform ops and complete gamma and poisson --- mindspore/ops/_op_impl/aicpu/gamma.py | 1 + mindspore/ops/_op_impl/aicpu/poisson.py | 1 + mindspore/ops/_op_impl/aicpu/uniform_int.py | 1 + mindspore/ops/_op_impl/aicpu/uniform_real.py | 7 +- mindspore/ops/composite/__init__.py | 4 +- mindspore/ops/composite/random_ops.py | 65 ++++++++++++++-- mindspore/ops/operations/random_ops.py | 77 ++++++++----------- .../ops/ascend/test_aicpu_ops/test_gamma.py | 10 +-- .../ops/ascend/test_aicpu_ops/test_normal.py | 3 +- .../ops/ascend/test_aicpu_ops/test_poisson.py | 8 +- .../test_aicpu_ops/test_standard_normal.py | 2 +- .../ops/ascend/test_aicpu_ops/test_uniform.py | 57 ++++++++++++++ .../ascend/test_aicpu_ops/test_uniform_int.py | 18 +---- .../test_aicpu_ops/test_uniform_real.py | 31 ++------ tests/ut/python/ops/test_ops.py | 29 ++----- 15 files changed, 180 insertions(+), 134 deletions(-) create mode 100644 tests/st/ops/ascend/test_aicpu_ops/test_uniform.py diff --git a/mindspore/ops/_op_impl/aicpu/gamma.py b/mindspore/ops/_op_impl/aicpu/gamma.py index b6b92a9da4..801ae41d6a 100644 --- a/mindspore/ops/_op_impl/aicpu/gamma.py +++ b/mindspore/ops/_op_impl/aicpu/gamma.py @@ -23,6 +23,7 @@ gamma_op_info = AiCPURegOp("Gamma") \ .input(2, "beta", "required") \ .output(0, "output", "required") \ .attr("seed", "int") \ + .attr("seed2", "int") \ .dtype_format(DataType.I32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ .dtype_format(DataType.I32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/aicpu/poisson.py b/mindspore/ops/_op_impl/aicpu/poisson.py index 59d2c1b957..4569efe40e 100644 --- a/mindspore/ops/_op_impl/aicpu/poisson.py +++ b/mindspore/ops/_op_impl/aicpu/poisson.py @@ -22,6 +22,7 @@ poisson_op_info = AiCPURegOp("Poisson") \ .input(1, "mean", "required") \ .output(0, "output", "required") \ .attr("seed", "int") \ + .attr("seed2", "int") \ .dtype_format(DataType.I32_Default, DataType.F32_Default, DataType.I32_Default) \ .dtype_format(DataType.I32_NCHW, DataType.F32_NCHW, DataType.I32_NCHW) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/aicpu/uniform_int.py b/mindspore/ops/_op_impl/aicpu/uniform_int.py index 35cfbec11c..3e76dc794a 100644 --- a/mindspore/ops/_op_impl/aicpu/uniform_int.py +++ b/mindspore/ops/_op_impl/aicpu/uniform_int.py @@ -23,6 +23,7 @@ uniform_int_op_info = AiCPURegOp("UniformInt") \ .input(2, "b", "required") \ .output(0, "output", "required") \ .attr("seed", "int") \ + .attr("seed2", "int") \ .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ .dtype_format(DataType.I32_NCHW, DataType.I32_NCHW, DataType.I32_NCHW, DataType.I32_NCHW) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/aicpu/uniform_real.py b/mindspore/ops/_op_impl/aicpu/uniform_real.py index 51824fbb2c..9e0876d317 100644 --- a/mindspore/ops/_op_impl/aicpu/uniform_real.py +++ b/mindspore/ops/_op_impl/aicpu/uniform_real.py @@ -19,12 +19,11 @@ from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataTyp uniform_real_op_info = AiCPURegOp("UniformReal") \ .fusion_type("OPAQUE") \ .input(0, "shape", "required") \ - .input(1, "a", "required") \ - .input(2, "b", "required") \ .output(0, "output", "required") \ .attr("seed", "int") \ - .dtype_format(DataType.I32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW) \ + .attr("seed2", "int") \ + .dtype_format(DataType.I32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_NCHW, DataType.F32_NCHW) \ .get_op_info() @op_info_register(uniform_real_op_info) diff --git a/mindspore/ops/composite/__init__.py b/mindspore/ops/composite/__init__.py index bb5e2960ff..ab35dd65fb 100644 --- a/mindspore/ops/composite/__init__.py +++ b/mindspore/ops/composite/__init__.py @@ -27,7 +27,7 @@ from .clip_ops import clip_by_value from .multitype_ops.add_impl import hyper_add from .multitype_ops.ones_like_impl import ones_like from .multitype_ops.zeros_like_impl import zeros_like -from .random_ops import normal +from .random_ops import set_seed, normal, uniform __all__ = [ @@ -48,5 +48,7 @@ __all__ = [ 'zeros_like', 'ones_like', 'zip_operation', + 'set_seed', + 'uniform', 'normal', 'clip_by_value',] diff --git a/mindspore/ops/composite/random_ops.py b/mindspore/ops/composite/random_ops.py index db338f5672..88037aefb7 100644 --- a/mindspore/ops/composite/random_ops.py +++ b/mindspore/ops/composite/random_ops.py @@ -13,10 +13,13 @@ # limitations under the License. # ============================================================================ -"""Operations for random number generatos.""" +"""Operations for random number generators.""" -from mindspore.ops.primitive import constexpr from .. import operations as P +from .. import functional as F +from ..primitive import constexpr +from .multitype_ops import _constexpr_utils as const_utils +from ...common import dtype as mstype # set graph-level RNG seed _GRAPH_SEED = 0 @@ -31,17 +34,17 @@ def get_seed(): return _GRAPH_SEED -def normal(shape, mean, stddev, seed): +def normal(shape, mean, stddev, seed=0): """ Generates random numbers according to the Normal (or Gaussian) random number distribution. It is defined as: Args: - - **shape** (tuple) - The shape of random tensor to be generated. - - **mean** (Tensor) - The mean μ distribution parameter, which specifies the location of the peak. + shape (tuple): The shape of random tensor to be generated. + mean (Tensor): The mean μ distribution parameter, which specifies the location of the peak. With float32 data type. - - **stddev** (Tensor) - The deviation σ distribution parameter. With float32 data type. - - **seed** (int): Seed is used as entropy source for Random number engines generating pseudo-random numbers. + stddev (Tensor): The deviation σ distribution parameter. With float32 data type. + seed (int): Seed is used as entropy source for Random number engines generating pseudo-random numbers. Default: 0. Returns: @@ -52,12 +55,58 @@ def normal(shape, mean, stddev, seed): >>> shape = (4, 16) >>> mean = Tensor(1.0, mstype.float32) >>> stddev = Tensor(1.0, mstype.float32) + >>> C.set_seed(10) >>> output = C.normal(shape, mean, stddev, seed=5) """ - set_seed(10) + mean_dtype = F.dtype(mean) + stddev_dtype = F.dtype(stddev) + const_utils.check_tensors_dtype_same(mean_dtype, mstype.float32, "normal") + const_utils.check_tensors_dtype_same(stddev_dtype, mstype.float32, "normal") seed1 = get_seed() seed2 = seed stdnormal = P.StandardNormal(seed1, seed2) rnd = stdnormal(shape) value = rnd * stddev + mean return value + +def uniform(shape, a, b, seed=0, dtype=mstype.float32): + """ + Generates random numbers according to the Uniform (or Gaussian) random number distribution. + It is defined as: + + Args: + shape (tuple): The shape of random tensor to be generated. + a (Tensor): The a distribution parameter. + It defines the minimum possibly generated value. With int32 or float32 data type. + If dtype is int32, only one number is allowed. + b (Tensor): The b distribution parameter. + It defines the maximum possibly generated value. With int32 or float32 data type. + If dtype is int32, only one number is allowed. + seed (int): Seed is used as entropy source for Random number engines generating pseudo-random numbers. + Default: 0. + + Returns: + Tensor. The shape should be the broadcasted shape of Input "shape" and shapes of a and b. + The dtype is float32. + + Examples: + >>> shape = (4, 16) + >>> a = Tensor(1.0, mstype.float32) + >>> b = Tensor(1.0, mstype.float32) + >>> C.set_seed(10) + >>> output = C.uniform(shape, a, b, seed=5) + """ + a_dtype = F.dtype(a) + b_dtype = F.dtype(b) + const_utils.check_tensors_dtype_same(a_dtype, dtype, "uniform") + const_utils.check_tensors_dtype_same(b_dtype, dtype, "uniform") + seed1 = get_seed() + seed2 = seed + if const_utils.is_same_type(dtype, mstype.int32): + rnd = P.UniformInt(seed1, seed2) + value = rnd(shape, a, b) + else: + uniform_real = P.UniformReal(seed1, seed2) + rnd = uniform_real(shape) + value = rnd * (b - a) + a + return value diff --git a/mindspore/ops/operations/random_ops.py b/mindspore/ops/operations/random_ops.py index d2c67b8f1b..59b28cf09d 100644 --- a/mindspore/ops/operations/random_ops.py +++ b/mindspore/ops/operations/random_ops.py @@ -34,8 +34,7 @@ class StandardNormal(PrimitiveWithInfer): - **shape** (tuple) - The shape of random tensor to be generated. Only constant value is allowed. Outputs: - Tensor. The shape should be the broadcasted shape of Input "shape" and shapes of mean and stddev. - The dtype is float32. + Tensor. The shape that the input 'shape' denotes. The dtype is float32. Examples: >>> shape = (4, 16) @@ -126,8 +125,8 @@ class Gamma(PrimitiveWithInfer): \text{P}(x|α,β) = \frac{\exp(-x/β)}{{β^α}\cdot{\Gamma(α)}}\cdot{x^{α-1}}, Args: - seed (int): Seed data is used as entropy source for Random number engines generating pseudo-random numbers. - Default: 0. + seed (int): Random seed. Default: 0. + seed2 (int): Random seed2. Default: 0. Inputs: - **shape** (tuple) - The shape of random tensor to be generated. Only constant value is allowed. @@ -149,10 +148,11 @@ class Gamma(PrimitiveWithInfer): """ @prim_attr_register - def __init__(self, seed=0): + def __init__(self, seed=0, seed2=0): """Init Gamma""" self.init_prim_io_names(inputs=['shape', 'alpha', 'beta'], outputs=['output']) validator.check_value_type('seed', seed, [int], self.name) + validator.check_value_type('seed2', seed2, [int], self.name) def __infer__(self, shape, alpha, beta): shape_v = shape["value"] @@ -180,8 +180,8 @@ class Poisson(PrimitiveWithInfer): \text{P}(i|μ) = \frac{\exp(-μ)μ^{i}}{i!}, Args: - seed (int): Seed data is used as entropy source for Random number engines generating pseudo-random numbers. - Default: 0. + seed (int): Random seed. Default: 0. + seed2 (int): Random seed2. Default: 0. Inputs: - **shape** (tuple) - The shape of random tensor to be generated. Only constant value is allowed. @@ -200,10 +200,11 @@ class Poisson(PrimitiveWithInfer): """ @prim_attr_register - def __init__(self, seed=0): + def __init__(self, seed=0, seed2=0): """Init Poisson""" self.init_prim_io_names(inputs=['shape', 'mean'], outputs=['output']) validator.check_value_type('seed', seed, [int], self.name) + validator.check_value_type('seed2', seed2, [int], self.name) def __infer__(self, shape, mean): shape_v = shape["value"] @@ -223,7 +224,7 @@ class Poisson(PrimitiveWithInfer): class UniformInt(PrimitiveWithInfer): r""" - Produces random integer values i, uniformly distributed on the closed interval [a, b], that is, + Produces random integer values i, uniformly distributed on the closed interval [a, b), that is, distributed according to the discrete probability function: .. math:: @@ -233,19 +234,18 @@ class UniformInt(PrimitiveWithInfer): The number in tensor a should be strictly less than b at any position after broadcasting. Args: - seed (int): Seed data is used as entropy source for Random number engines generating pseudo-random numbers. - Default: 0. + seed (int): Random seed. Default: 0. + seed2 (int): Random seed2. Default: 0. Inputs: - **shape** (tuple) - The shape of random tensor to be generated. Only constant value is allowed. - **a** (Tensor) - The a distribution parameter. - It defines the minimum possibly generated value. With int32 data type. + It defines the minimum possibly generated value. With int32 data type. Only one number is supported. - **b** (Tensor) - The b distribution parameter. - It defines the maximum possibly generated value. With int32 data type. + It defines the maximum possibly generated value. With int32 data type. Only one number is supported. Outputs: - Tensor. The shape should be the broadcasted shape of Input "shape" and shapes of a and b. - The dtype is int32. + Tensor. The shape that the input 'shape' denotes. The dtype is int32. Examples: >>> shape = (4, 16) @@ -256,10 +256,11 @@ class UniformInt(PrimitiveWithInfer): """ @prim_attr_register - def __init__(self, seed=0): + def __init__(self, seed=0, seed2=0): """Init UniformInt""" self.init_prim_io_names(inputs=['shape', 'a', 'b'], outputs=['output']) validator.check_value_type('seed', seed, [int], self.name) + validator.check_value_type('seed2', seed2, [int], self.name) def __infer__(self, shape, a, b): shape_v = shape["value"] @@ -270,10 +271,12 @@ class UniformInt(PrimitiveWithInfer): validator.check_integer("shape[%d]" % i, shape_i, 0, Rel.GT, self.name) validator.check_tensor_type_same({"a": a["dtype"]}, [mstype.int32], self.name) validator.check_tensor_type_same({"b": b["dtype"]}, [mstype.int32], self.name) - broadcast_shape = get_broadcast_shape(a['shape'], b['shape'], self.name) - broadcast_shape = get_broadcast_shape(broadcast_shape, shape_v, self.name) + a_shape = a['shape'] + b_shape = b['shape'] + validator.check("dim of a", len(a_shape), '0(scalar)', 0, Rel.EQ, self.name) + validator.check("dim of b", len(b_shape), '0(scalar)', 0, Rel.EQ, self.name) out = { - 'shape': broadcast_shape, + 'shape': shape_v, 'dtype': mstype.int32, 'value': None} return out @@ -281,54 +284,40 @@ class UniformInt(PrimitiveWithInfer): class UniformReal(PrimitiveWithInfer): r""" - Produces random floating-point values i, uniformly distributed on the interval [min(a, b), max(a, b)), that is,\ - distributed according to the probability density function: - - .. math:: - \text{P}(i|a,b) = \frac{1}{b-a}, + Produces random floating-point values i, uniformly distributed on the interval [0, 1). Args: - seed (int): Seed data is used as entropy source for Random number engines generating pseudo-random numbers. - Default: 0. + seed (int): Random seed. Default: 0. + seed2 (int): Random seed2. Default: 0. Inputs: - **shape** (tuple) - The shape of random tensor to be generated. Only constant value is allowed. - - **a** (Tensor) - The a distribution parameter. - It defines the minimum possibly generated value. With float32 data type. - - **b** (Tensor) - The b distribution parameter. - It defines the maximum possibly generated value. With float32 data type. Outputs: - Tensor. The shape should be the broadcasted shape of Input "shape" and shapes of a and b. - The dtype is float32. + Tensor. The shape that the input 'shape' denotes. The dtype is float32. Examples: >>> shape = (4, 16) - >>> a = Tensor(1.0, mstype.float32) - >>> b = Tensor(5.0, mstype.float32) - >>> uniform_real = P.UniformReal(seed=10) - >>> output = uniform_real(shape, a, b) + >>> uniformreal = P.UniformReal(seed=2) + >>> output = uniformreal(shape) """ @prim_attr_register - def __init__(self, seed=0): + def __init__(self, seed=0, seed2=0): """Init UniformReal""" - self.init_prim_io_names(inputs=['shape', 'a', 'b'], outputs=['output']) + self.init_prim_io_names(inputs=['shape'], outputs=['output']) validator.check_value_type('seed', seed, [int], self.name) + validator.check_value_type('seed2', seed2, [int], self.name) - def __infer__(self, shape, a, b): + def __infer__(self, shape): shape_v = shape["value"] if shape_v is None: raise ValueError(f"For {self.name}, shape must be const.") validator.check_value_type("shape", shape_v, [tuple], self.name) for i, shape_i in enumerate(shape_v): validator.check_integer("shape[%d]" % i, shape_i, 0, Rel.GT, self.name) - validator.check_tensor_type_same({"a": a["dtype"]}, [mstype.float32], self.name) - validator.check_tensor_type_same({"b": b["dtype"]}, [mstype.float32], self.name) - broadcast_shape = get_broadcast_shape(a['shape'], b['shape'], self.name) - broadcast_shape = get_broadcast_shape(broadcast_shape, shape_v, self.name) out = { - 'shape': broadcast_shape, + 'shape': shape_v, 'dtype': mstype.float32, 'value': None} return out diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_gamma.py b/tests/st/ops/ascend/test_aicpu_ops/test_gamma.py index 4b685df16b..61bb3f8476 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_gamma.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_gamma.py @@ -24,9 +24,9 @@ context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") class Net(nn.Cell): - def __init__(self, shape, seed=0): + def __init__(self, shape, seed=0, seed2=0): super(Net, self).__init__() - self.gamma = P.Gamma(seed=seed) + self.gamma = P.Gamma(seed=seed, seed2=seed2) self.shape = shape def construct(self, alpha, beta): @@ -38,10 +38,9 @@ def test_net_1D(): shape = (3, 2, 4) alpha = 1.0 beta = 1.0 - net = Net(shape, seed) + net = Net(shape=shape, seed=seed) talpha, tbeta = Tensor(alpha, mstype.float32), Tensor(beta, mstype.float32) output = net(talpha, tbeta) - print(output.asnumpy()) assert output.shape == (3, 2, 4) @@ -50,9 +49,8 @@ def test_net_ND(): shape = (3, 1, 2) alpha = np.array([[[1], [2]], [[3], [4]], [[5], [6]]]).astype(np.float32) beta = np.array([1.0]).astype(np.float32) - net = Net(shape, seed) + net = Net(shape=shape, seed=seed) talpha, tbeta = Tensor(alpha), Tensor(beta) output = net(talpha, tbeta) - print(output.asnumpy()) assert output.shape == (3, 2, 2) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_normal.py b/tests/st/ops/ascend/test_aicpu_ops/test_normal.py index 77f0b7e8e8..70c9c2c68f 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_normal.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_normal.py @@ -32,6 +32,7 @@ class Net(nn.Cell): self.seed = seed def construct(self, mean, stddev): + C.set_seed(20) return C.normal(self.shape, mean, stddev, self.seed) @@ -43,7 +44,6 @@ def test_net_1D(): net = Net(shape, seed) tmean, tstddev = Tensor(mean, mstype.float32), Tensor(stddev, mstype.float32) output = net(tmean, tstddev) - print(output.asnumpy()) assert output.shape == (3, 2, 4) @@ -55,5 +55,4 @@ def test_net_ND(): net = Net(shape, seed) tmean, tstddev = Tensor(mean, mstype.float32), Tensor(stddev, mstype.float32) output = net(tmean, tstddev) - print(output.asnumpy()) assert output.shape == (3, 2, 2) \ No newline at end of file diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py b/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py index 29af6cbb09..dd5ada2712 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py @@ -24,7 +24,7 @@ context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") class Net(nn.Cell): - def __init__(self, shape): + def __init__(self, shape, seed=0, seed2=0): super(Net, self).__init__() self.poisson = P.Poisson() self.shape = shape @@ -36,18 +36,16 @@ class Net(nn.Cell): def test_net_1(): shape = (2, 16) mean = np.array([5.0]).astype(np.float32) - net = Net(shape) + net = Net(shape=shape) tmean = Tensor(mean) output = net(tmean) - print(output.asnumpy()) assert output.shape == (2, 16) def test_net_2(): shape = (4, 1) mean = np.array([5.0, 10.0]).astype(np.float32) - net = Net(shape) + net = Net(shape=shape) tmean = Tensor(mean) output = net(tmean) - print(output.asnumpy()) assert output.shape == (4, 2) \ No newline at end of file diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_standard_normal.py b/tests/st/ops/ascend/test_aicpu_ops/test_standard_normal.py index 5cc21fac80..f45cb462cc 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_standard_normal.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_standard_normal.py @@ -34,7 +34,7 @@ class Net(nn.Cell): self.stdnormal = P.StandardNormal(seed, seed2) def construct(self): - return self.stdnormal(self.shape, self.seed, self.seed2) + return self.stdnormal(self.shape) def test_net(): diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_uniform.py b/tests/st/ops/ascend/test_aicpu_ops/test_uniform.py new file mode 100644 index 0000000000..cef50bdbc7 --- /dev/null +++ b/tests/st/ops/ascend/test_aicpu_ops/test_uniform.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np + +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.ops import composite as C + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, shape, seed=0): + super(Net, self).__init__() + self.shape = shape + self.seed = seed + + def construct(self, a, b): + C.set_seed(20) + return C.uniform(self.shape, a, b, self.seed) + + +def test_net_1D(): + seed = 10 + shape = (3, 2, 4) + a = 1.0 + b = 6.0 + net = Net(shape, seed) + ta, tb = Tensor(a, mstype.float32), Tensor(b, mstype.float32) + output = net(ta, tb) + assert output.shape == (3, 2, 4) + + +def test_net_ND(): + seed = 10 + shape = (3, 1, 2) + a = np.array([[[1], [2]], [[3], [4]], [[5], [6]]]).astype(np.float32) + b = np.array([1.0]).astype(np.float32) + net = Net(shape, seed) + ta, tb = Tensor(a, mstype.float32), Tensor(b, mstype.float32) + output = net(ta, tb) + assert output.shape == (3, 2, 2) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_uniform_int.py b/tests/st/ops/ascend/test_aicpu_ops/test_uniform_int.py index cbd39f4706..fef34fad07 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_uniform_int.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_uniform_int.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np import mindspore.context as context import mindspore.nn as nn @@ -24,7 +23,7 @@ context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") class Net(nn.Cell): - def __init__(self, shape, seed=0): + def __init__(self, shape, seed=0, seed2=0): super(Net, self).__init__() self.uniformint = P.UniformInt(seed=seed) self.shape = shape @@ -38,20 +37,7 @@ def test_net_1D(): shape = (3, 2, 4) a = 1 b = 5 - net = Net(shape, seed) + net = Net(shape, seed=seed) ta, tb = Tensor(a, mstype.int32), Tensor(b, mstype.int32) output = net(ta, tb) - print(output.asnumpy()) assert output.shape == (3, 2, 4) - - -def test_net_ND(): - seed = 10 - shape = (3, 2, 1) - a = np.array([[[1, 2]], [[3, 4]], [[5, 6]]]).astype(np.int32) - b = np.array([10]).astype(np.int32) - net = Net(shape, seed) - ta, tb = Tensor(a), Tensor(b) - output = net(ta, tb) - print(output.asnumpy()) - assert output.shape == (3, 2, 2) \ No newline at end of file diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_uniform_real.py b/tests/st/ops/ascend/test_aicpu_ops/test_uniform_real.py index 635eb3fa28..7ab0b42e11 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_uniform_real.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_uniform_real.py @@ -12,46 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np import mindspore.context as context import mindspore.nn as nn -from mindspore import Tensor from mindspore.ops import operations as P -from mindspore.common import dtype as mstype context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") class Net(nn.Cell): - def __init__(self, shape, seed=0): + def __init__(self, shape, seed=0, seed2=0): super(Net, self).__init__() self.uniformreal = P.UniformReal(seed=seed) self.shape = shape - def construct(self, a, b): - return self.uniformreal(self.shape, a, b) + def construct(self): + return self.uniformreal(self.shape) -def test_net_1D(): +def test_net(): seed = 10 shape = (3, 2, 4) - a = 1.0 - b = 5.0 - net = Net(shape, seed) - ta, tb = Tensor(a, mstype.float32), Tensor(b, mstype.float32) - output = net(ta, tb) - print(output.asnumpy()) + net = Net(shape, seed=seed) + output = net() assert output.shape == (3, 2, 4) - - -def test_net_ND(): - seed = 10 - shape = (3, 2, 1) - a = np.array([[[1, 2]], [[3, 4]], [[5, 6]]]).astype(np.float32) - b = np.array([10]).astype(np.float32) - net = Net(shape, seed) - ta, tb = Tensor(a), Tensor(b) - output = net(ta, tb) - print(output.asnumpy()) - assert output.shape == (3, 2, 2) \ No newline at end of file diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index 31ca540f74..7f53d40469 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -573,25 +573,14 @@ class PoissonNet(nn.Cell): return out -class UniformIntNet(nn.Cell): +class UniformNet(nn.Cell): def __init__(self, shape=None, seed=0): - super(UniformIntNet, self).__init__() - self.uniformint = P.UniformInt(seed=seed) - self.shape = shape - - def construct(self, a, b): - out = self.uniformint(self.shape, a, b) - return out - - -class UniformRealNet(nn.Cell): - def __init__(self, shape=None, seed=0): - super(UniformRealNet, self).__init__() - self.uniformreal = P.UniformReal(seed=seed) + super(UniformNet, self).__init__() self.shape = shape + self.seed = seed def construct(self, a, b): - out = self.uniformreal(self.shape, a, b) + out = C.uniform(self.shape, a, b, self.seed) return out @@ -882,13 +871,9 @@ test_case_math_ops = [ 'block': PoissonNet((3, 2, 4), 0), 'desc_inputs': [Tensor(2.0, mstype.float32)], 'skip': ['backward']}), - ('UniformInt', { - 'block': UniformIntNet((3, 2, 4), 0), - 'desc_inputs': [Tensor(1, mstype.int32), Tensor(15, mstype.int32)], - 'skip': ['backward']}), - ('UniformReal', { - 'block': UniformRealNet((3, 2, 4), 0), - 'desc_inputs': [Tensor(1.0, mstype.float32), Tensor(5.0, mstype.float32)], + ('Uniform', { + 'block': UniformNet((3, 2, 4), 0), + 'desc_inputs': [Tensor(0.0, mstype.float32), Tensor(1.0, mstype.float32)], 'skip': ['backward']}), ('RandomChoiceWithMask', { 'block': P.RandomChoiceWithMask(256), From 5a5703768c56d50d1b68feb43e18b2d20221c0df Mon Sep 17 00:00:00 2001 From: hanjun996 Date: Fri, 31 Jul 2020 11:39:23 +0800 Subject: [PATCH 10/13] modify tdt open and close --- mindspore/ccsrc/utils/context/ms_context.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index 720c77e23e..79f5b147ac 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -192,7 +192,7 @@ bool MsContext::OpenTsd() { } MS_LOG(INFO) << "Device id = " << device_id << ", rank size = " << rank_size << "."; - TDT_StatusT status = tdt::TsdClient::GetInstance()->Open(device_id, rank_size); + TDT_StatusT status = TsdOpen(device_id, rank_size); if (status != TDT_OK) { MS_LOG(EXCEPTION) << "Device " << device_id << " is occupied, open tsd failed, status = " << status << "."; return false; @@ -238,7 +238,9 @@ bool MsContext::CloseTsd(bool force) { MS_LOG(ERROR) << "tdt thread join failed: " << e.what(); } #endif - TDT_StatusT status = tdt::TsdClient::GetInstance()->Close(); + unsigned int device_id; + device_id = device_id_; + TDT_StatusT status = TsdClose(device_id); if (status != TDT_OK) { MS_LOG(EXCEPTION) << "Close tsd failed, status = " << status << "."; return false; From 3820533ad176ef82f82e6f5f22eec618d5ad15d0 Mon Sep 17 00:00:00 2001 From: peixu_ren Date: Fri, 7 Aug 2020 14:41:53 -0400 Subject: [PATCH 11/13] Refactor Gamma and Poisson ops --- mindspore/ops/composite/__init__.py | 6 +- mindspore/ops/composite/random_ops.py | 64 +++++++++++++++++-- .../test_compoite_random_ops/test_gamma.py | 56 ++++++++++++++++ .../test_normal.py | 3 - .../test_compoite_random_ops/test_poisson.py | 54 ++++++++++++++++ .../test_uniform.py | 1 - tests/ut/python/ops/test_ops.py | 8 +-- 7 files changed, 177 insertions(+), 15 deletions(-) create mode 100644 tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py rename tests/st/ops/ascend/{test_aicpu_ops => test_compoite_random_ops}/test_normal.py (99%) create mode 100644 tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py rename tests/st/ops/ascend/{test_aicpu_ops => test_compoite_random_ops}/test_uniform.py (99%) diff --git a/mindspore/ops/composite/__init__.py b/mindspore/ops/composite/__init__.py index 6656dafdb4..b06a2a397e 100644 --- a/mindspore/ops/composite/__init__.py +++ b/mindspore/ops/composite/__init__.py @@ -27,7 +27,7 @@ from .clip_ops import clip_by_value from .multitype_ops.add_impl import hyper_add from .multitype_ops.ones_like_impl import ones_like from .multitype_ops.zeros_like_impl import zeros_like -from .random_ops import set_seed, normal, multinomial, uniform +from .random_ops import set_seed, normal, uniform, gamma, poisson, multinomial __all__ = [ @@ -49,7 +49,9 @@ __all__ = [ 'ones_like', 'zip_operation', 'set_seed', - 'uniform', 'normal', + 'uniform', + 'gamma', + 'poisson', 'multinomial', 'clip_by_value',] diff --git a/mindspore/ops/composite/random_ops.py b/mindspore/ops/composite/random_ops.py index 70313e72e0..b052e7a795 100644 --- a/mindspore/ops/composite/random_ops.py +++ b/mindspore/ops/composite/random_ops.py @@ -66,7 +66,6 @@ def get_seed(): def normal(shape, mean, stddev, seed=0): """ Generates random numbers according to the Normal (or Gaussian) random number distribution. - It is defined as: Args: shape (tuple): The shape of random tensor to be generated. @@ -84,7 +83,6 @@ def normal(shape, mean, stddev, seed=0): >>> shape = (4, 16) >>> mean = Tensor(1.0, mstype.float32) >>> stddev = Tensor(1.0, mstype.float32) - >>> C.set_seed(10) >>> output = C.normal(shape, mean, stddev, seed=5) """ mean_dtype = F.dtype(mean) @@ -148,8 +146,7 @@ def multinomial(inputs, num_sample=None, replacement=True, seed=0): def uniform(shape, a, b, seed=0, dtype=mstype.float32): """ - Generates random numbers according to the Uniform (or Gaussian) random number distribution. - It is defined as: + Generates random numbers according to the Uniform random number distribution. Args: shape (tuple): The shape of random tensor to be generated. @@ -170,7 +167,6 @@ def uniform(shape, a, b, seed=0, dtype=mstype.float32): >>> shape = (4, 16) >>> a = Tensor(1.0, mstype.float32) >>> b = Tensor(1.0, mstype.float32) - >>> C.set_seed(10) >>> output = C.uniform(shape, a, b, seed=5) """ a_dtype = F.dtype(a) @@ -187,3 +183,61 @@ def uniform(shape, a, b, seed=0, dtype=mstype.float32): rnd = uniform_real(shape) value = rnd * (b - a) + a return value + +def gamma(shape, alpha, beta, seed=0): + """ + Generates random numbers according to the Gamma random number distribution. + + Args: + shape (tuple): The shape of random tensor to be generated. + alpha (Tensor): The alpha α distribution parameter. With float32 data type. + beta (Tensor): The beta β distribution parameter. With float32 data type. + seed (int): Seed is used as entropy source for Random number engines generating pseudo-random numbers. + Default: 0. + + Returns: + Tensor. The shape should be the broadcasted shape of Input "shape" and shapes of alpha and beta. + The dtype is float32. + + Examples: + >>> shape = (4, 16) + >>> alpha = Tensor(1.0, mstype.float32) + >>> beta = Tensor(1.0, mstype.float32) + >>> output = C.gamma(shape, alpha, beta, seed=5) + """ + alpha_dtype = F.dtype(alpha) + beta_dtype = F.dtype(beta) + const_utils.check_tensors_dtype_same(alpha_dtype, mstype.float32, "gamma") + const_utils.check_tensors_dtype_same(beta_dtype, mstype.float32, "gamma") + seed1 = get_seed() + seed2 = seed + gamma = P.Gamma(seed1, seed2) + value = gamma(shape, alpha, beta) + return value + +def poisson(shape, mean, seed=0): + """ + Generates random numbers according to the Poisson random number distribution. + + Args: + shape (tuple): The shape of random tensor to be generated. + mean (Tensor): The mean μ distribution parameter. With float32 data type. + seed (int): Seed is used as entropy source for Random number engines generating pseudo-random numbers. + Default: 0. + + Returns: + Tensor. The shape should be the broadcasted shape of Input "shape" and shapes of mean. + The dtype is float32. + + Examples: + >>> shape = (4, 16) + >>> mean = Tensor(1.0, mstype.float32) + >>> output = C.poisson(shape, mean, seed=5) + """ + mean_dtype = F.dtype(mean) + const_utils.check_tensors_dtype_same(mean_dtype, mstype.float32, "poisson") + seed1 = get_seed() + seed2 = seed + poisson = P.Poisson(seed1, seed2) + value = poisson(shape, mean) + return value diff --git a/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py b/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py new file mode 100644 index 0000000000..e762aedc21 --- /dev/null +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np + +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.ops import composite as C + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, shape, seed=0): + super(Net, self).__init__() + self.shape = shape + self.seed = seed + + def construct(self, alpha, beta): + C.set_seed(20) + return C.gamma(self.shape, alpha, beta, self.seed) + + +def test_net_1D(): + seed = 10 + shape = (3, 2, 4) + alpha = 1.0 + beta = 1.0 + net = Net(shape, seed) + talpha, tbeta = Tensor(alpha, mstype.float32), Tensor(beta, mstype.float32) + output = net(talpha, tbeta) + assert output.shape == (3, 2, 4) + + +def test_net_ND(): + seed = 10 + shape = (3, 1, 2) + alpha = np.array([[[1], [2]], [[3], [4]], [[5], [6]]]).astype(np.float32) + beta = np.array([1.0]).astype(np.float32) + net = Net(shape, seed) + talpha, tbeta = Tensor(alpha, mstype.float32), Tensor(beta, mstype.float32) + output = net(talpha, tbeta) + assert output.shape == (3, 2, 2) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_normal.py b/tests/st/ops/ascend/test_compoite_random_ops/test_normal.py similarity index 99% rename from tests/st/ops/ascend/test_aicpu_ops/test_normal.py rename to tests/st/ops/ascend/test_compoite_random_ops/test_normal.py index 01ecbf5ec4..6c6e07b584 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_normal.py +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_normal.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - import numpy as np -import pytest import mindspore.context as context import mindspore.nn as nn @@ -56,4 +54,3 @@ def test_net_ND(): tmean, tstddev = Tensor(mean, mstype.float32), Tensor(stddev, mstype.float32) output = net(tmean, tstddev) assert output.shape == (3, 2, 2) - diff --git a/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py b/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py new file mode 100644 index 0000000000..caa0a1f642 --- /dev/null +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np + +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.ops import composite as C + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, shape, seed=0): + super(Net, self).__init__() + self.shape = shape + self.seed = seed + + def construct(self, mean): + C.set_seed(20) + return C.poisson(self.shape, mean, self.seed) + + +def test_net_1D(): + seed = 10 + shape = (3, 2, 4) + mean = 1.0 + net = Net(shape, seed) + tmean = Tensor(mean, mstype.float32) + output = net(tmean) + assert output.shape == (3, 2, 4) + + +def test_net_ND(): + seed = 10 + shape = (3, 1, 2) + mean = np.array([[[1], [2]], [[3], [4]], [[5], [6]]]).astype(np.float32) + net = Net(shape, seed) + tmean = Tensor(mean, mstype.float32) + output = net(tmean) + assert output.shape == (3, 2, 2) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_uniform.py b/tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py similarity index 99% rename from tests/st/ops/ascend/test_aicpu_ops/test_uniform.py rename to tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py index cef50bdbc7..44a7798250 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_uniform.py +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - import numpy as np import mindspore.context as context diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index 8b2e7ab432..2eb3584c33 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -592,22 +592,22 @@ class LaplaceNet(nn.Cell): class GammaNet(nn.Cell): def __init__(self, shape=None, seed=0): super(GammaNet, self).__init__() - self.gamma = P.Gamma(seed=seed) self.shape = shape + self.seed = seed def construct(self, alpha, beta): - out = self.gamma(self.shape, alpha, beta) + out = C.gamma(self.shape, alpha, beta, self.seed) return out class PoissonNet(nn.Cell): def __init__(self, shape=None, seed=0): super(PoissonNet, self).__init__() - self.poisson = P.Poisson(seed=seed) self.shape = shape + self.seed = seed def construct(self, mean): - out = self.poisson(self.shape, mean) + out = C.poisson(self.shape, mean, self.seed) return out From 5f7fd2ea77549fa17e45d310ef11af36c7cecae6 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Tue, 11 Aug 2020 16:41:30 +0800 Subject: [PATCH 12/13] update incubator to run: C75B100 --- graphengine | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphengine b/graphengine index e64a1cfc04..377b216518 160000 --- a/graphengine +++ b/graphengine @@ -1 +1 @@ -Subproject commit e64a1cfc0457c96859bc9be1693443aa14f2e9df +Subproject commit 377b2165184fbfbb32829266822438e439861f14 From 866e9259a494f6b517046ae53d20979ad822a46a Mon Sep 17 00:00:00 2001 From: jonyguo Date: Tue, 11 Aug 2020 19:49:30 +0800 Subject: [PATCH 13/13] fix: random_op.py pylint error --- mindspore/ops/composite/random_ops.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mindspore/ops/composite/random_ops.py b/mindspore/ops/composite/random_ops.py index b052e7a795..0be6923a2e 100644 --- a/mindspore/ops/composite/random_ops.py +++ b/mindspore/ops/composite/random_ops.py @@ -136,10 +136,10 @@ def multinomial(inputs, num_sample=None, replacement=True, seed=0): n_dist = shape(inputs)[-2] a = Tensor(0.0, mstype.float32) b = Tensor(1.0, mstype.float32) - uniform = P.UniformReal(seed=seed)((n_dist * num_sample,), a, b) + random_uniform = P.UniformReal(seed=seed)((n_dist * num_sample,), a, b) if n_dist != 1: - uniform = reshape(uniform, (n_dist, num_sample)) - vals = P.RealDiv()(P.Log()(uniform), inputs + 1e-6) + random_uniform = reshape(random_uniform, (n_dist, num_sample)) + vals = P.RealDiv()(P.Log()(random_uniform), inputs + 1e-6) _, indices = P.TopK()(vals, num_sample) return indices return P.Multinomial(seed=seed)(inputs, num_sample) @@ -211,8 +211,8 @@ def gamma(shape, alpha, beta, seed=0): const_utils.check_tensors_dtype_same(beta_dtype, mstype.float32, "gamma") seed1 = get_seed() seed2 = seed - gamma = P.Gamma(seed1, seed2) - value = gamma(shape, alpha, beta) + random_gamma = P.Gamma(seed1, seed2) + value = random_gamma(shape, alpha, beta) return value def poisson(shape, mean, seed=0): @@ -238,6 +238,6 @@ def poisson(shape, mean, seed=0): const_utils.check_tensors_dtype_same(mean_dtype, mstype.float32, "poisson") seed1 = get_seed() seed2 = seed - poisson = P.Poisson(seed1, seed2) - value = poisson(shape, mean) + random_poisson = P.Poisson(seed1, seed2) + value = random_poisson(shape, mean) return value