diff --git a/mindspore/nn/optim/ftrl.py b/mindspore/nn/optim/ftrl.py index 93b2defe2d..942b31065c 100644 --- a/mindspore/nn/optim/ftrl.py +++ b/mindspore/nn/optim/ftrl.py @@ -13,13 +13,15 @@ # limitations under the License. # ============================================================================ """FTRL""" +# 本文件为FTRL优化器的构建 from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel -from .optimizer import Optimizer, _apply_decay, _grad_scale +from.optimizer import Optimizer, _apply_decay, _grad_scale +# 定义_ftrl_opt函数，用于接收四个参数：opt（优化器），spars_opt（模式优化器），push（推送），pull（拉取），l1（L1正则化），l2（L2正则化），lr_power（学习率指数），learning_rate（学习率），linear（线性），gradient（梯度），weight（权重），moment（动量），ps_parameter（参数），cache_enable（是否使用缓存） _ftrl_opt = C.MultitypeFuncGraph("ftrl_opt") @@ -28,14 +30,23 @@ _ftrl_opt = C.MultitypeFuncGraph("ftrl_opt") def _tensor_run_opt_with_sparse(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear, gradient, weight, moment, ps_parameter, cache_enable): """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse.""" + # 对为稀疏矩阵的参数权重使用FTRL优化器 + # 判断是否满足条件 success = True + # 获取梯度的索引和值 indices = gradient.indices values = gradient.values + # 如果没有指定ps参数，且不满足cache_enable条件 if ps_parameter and not cache_enable: + # 获取操作的形状 op_shape = P.Shape() + # 获取形状 shapes = (op_shape(weight), op_shape(moment), op_shape(linear), op_shape(values), op_shape(indices)) + # 执行pull操作 success = F.depend(success, pull(push((values, indices), shapes), weight)) + # 如果指定了ps参数，且满足cache_enable条件 else: + # 执行spars_opt操作 success = F.depend(success, spars_opt(weight, moment, linear, values, indices)) return success @@ -44,35 +55,52 @@ def _tensor_run_opt_with_sparse(opt, spars_opt, push, pull, l1, l2, lr_power, le "Tensor", "Tensor", "Tensor", "Bool", "Bool") def _tensor_run_opt(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear, gradient, weight, moment, ps_parameter, cache_enable): + # 对权重参数应用FTRL优化器 """Apply ftrl optimizer to the weight parameter.""" success = True + # 如果ps_parameter为True，且不支持缓存，则使用pull函数拉取push函数的输出 if ps_parameter and not cache_enable: op_shape = P.Shape() success = F.depend(success, pull(push((gradient, learning_rate, l1, l2, lr_power), (op_shape(weight), op_shape(moment), op_shape(linear))), weight)) + # 否则，使用opt函数计算weight的梯度，并使用push函数推送梯度 else: success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power)) + # 返回更新成功标志 return success def _check_param(initial_accum, lr_power, l1, l2, use_locking, prim_name=None): """Check param.""" + """ + 检查参数 + """ + # 检查initial_accum的类型是否为float validator.check_value_type("initial_accum", initial_accum, [float], prim_name) + # 检查initial_accum的值是否在0.0以上 validator.check_number("initial_accum", initial_accum, 0.0, Rel.GE, prim_name) + # 检查lr_power的类型是否为float validator.check_value_type("lr_power", lr_power, [float], prim_name) + # 检查lr_power的值是否在0.0和Rel.LE之间 validator.check_number("lr_power", lr_power, 0.0, Rel.LE, prim_name) + # 检查l1的类型是否为float validator.check_value_type("l1", l1, [float], prim_name) + # 检查l1的值是否在0.0和Rel.GE之间 validator.check_number("l1", l1, 0.0, Rel.GE, prim_name) + # 检查l2的类型是否为float validator.check_value_type("l2", l2, [float], prim_name) + # 检查l2的值是否在0.0和Rel.GE之间 validator.check_number("l2", l2, 0.0, Rel.GE, prim_name) + # 检查use_locking的类型是否为bool validator.check_value_type("use_locking", use_locking, [bool], prim_name) class FTRL(Optimizer): + # FTRL优化器 r""" Implements the FTRL algorithm with ApplyFtrl Operator. @@ -193,59 +221,89 @@ class FTRL(Optimizer): """ def __init__(self, params, initial_accum=0.1, learning_rate=0.001, lr_power=-0.5, l1=0.0, l2=0.0, use_locking=False, loss_scale=1.0, weight_decay=0.0): + # 初始化FTRL类 super(FTRL, self).__init__(learning_rate, params, weight_decay, loss_scale=loss_scale) + # 检查参数 if self.dynamic_lr or self.is_group_lr: raise ValueError('Dynamic learning rate or group learning rate is currently not supported.') _check_param(initial_accum, lr_power, l1, l2, use_locking, self.cls_name) + # 初始化参数 self.moments = self.parameters.clone(prefix="moments", init=initial_accum) self.linear = self.parameters.clone(prefix="linear", init='zeros') self.l1 = l1 self.l2 = l2 self.lr = learning_rate self.lr_power = lr_power + # 判断是否是分组 if not self.is_group: + # 如果不是组，则将decay_flags设置为True self.decay_flags = tuple((lambda: True)() for x in self.parameters) + # 初始化优化器 self.hyper_map = C.HyperMap() self.opt = P.ApplyFtrl(use_locking=use_locking) self.use_locking = use_locking + # 初始化sparse优化器 self.sparse_opt = P.SparseApplyFtrl(learning_rate, l1, l2, lr_power, use_locking=use_locking) + # 初始化sparse_opt self._ps_pull = P.Pull() + # 初始化_ps_pull self._ps_push = P.Push("Ftrl", [0, 1, 2]) + # 初始化_ps_push self._ps_push.add_prim_attr("init_accum", initial_accum) + # 向_ps_push中添加init_accum属性 self._ps_push.add_prim_attr("lr", learning_rate) + # 向_ps_push中添加lr属性 self._ps_push.add_prim_attr("l1", l1) + # 向_ps_push中添加l1属性 self._ps_push.add_prim_attr("l2", l2) + # 向_ps_push中添加l2属性 self._ps_push.add_prim_attr("lr_power", lr_power) def construct(self, grads): + ''' + 构建FTRL优化器 + :param grads: 梯度 + :return: 更新后的参数 + ''' params = self.parameters moments = self.moments linear = self.linear + # 对梯度权重衰减 grads = self.decay_weight(grads) + # 对梯度中心化 grads = self.gradients_centralization(grads) + # 对梯度进行缩放 grads = self.scale_grad(grads) + # 对梯度进行去重 grads = self._grad_sparse_indices_deduplicate(grads) + # 更新参数 lr = self.get_lr() + # 将_ftrl_opt函数放入参数 success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull, - self.l1, self.l2, self.lr_power, lr), - linear, grads, params, moments, self.ps_parameters, self.cache_enable) + self.l1, self.l2, self.lr_power, lr), + linear, grads, params, moments, self.ps_parameters, self.cache_enable) return success @Optimizer.target.setter + # 优化器target设置 def target(self, value): """If the input value is set to "CPU", the parameters will be updated on the host using the Fused optimizer operation.""" + # 如果value不为str类，则抛出TypeError异常 if not isinstance(value, str): raise TypeError("The value must be str type, but got value type is {}".format(type(value))) - + # 如果value值不在三者之中，则抛出ValueError异常 if value not in ('CPU', 'Ascend', 'GPU'): raise ValueError("The value must be 'CPU', 'Ascend' or 'GPU', but got value {}".format(value)) if value == 'CPU': + # 如果输入值为CPU，则使用FusedSparseFtrl优化器更新参数 self.sparse_opt = P.FusedSparseFtrl(self.lr, self.l1, self.l2, self.lr_power, self.use_locking) self.sparse_opt.add_prim_attr("primitive_target", "CPU") else: + # 如果输入值为GPU，则使用SparseApplyFtrl优化器更新参数 self.sparse_opt = P.SparseApplyFtrl(self.lr, self.l1, self.l2, self.lr_power, self.use_locking) - self._target = value + # 设置目标值 + self._target = value \ No newline at end of file