diff --git a/mindspore/nn/learning_rate_schedule.py b/mindspore/nn/learning_rate_schedule.py index a83cf9967b..5bfc7ac831 100644 --- a/mindspore/nn/learning_rate_schedule.py +++ b/mindspore/nn/learning_rate_schedule.py @@ -13,17 +13,22 @@ # limitations under the License. # ============================================================================ """Learning rate schedule.""" - +# 本文件为动态学习率的定义 +# 导入数学模块 import math - +# 导入数据类型模块 from ..common import dtype as mstype +# 导入ops算子 from ..ops import operations as P +# 导入神经网络基本单元Cell from .cell import Cell +# 导入检查模块 from .._checkparam import Validator as validator class LearningRateSchedule(Cell): """Basic class of learning rate schedule.""" + # LearningRateSchedule的基本类，所有动态学习率均继承自此类 def __init__(self): super(LearningRateSchedule, self).__init__() @@ -43,6 +48,15 @@ class LearningRateSchedule(Cell): def _check_inputs(learning_rate, decay_rate, decay_steps, is_stair, cls_name): + ''' + 检查输入参数是否合法 + :param learning_rate: 学习率 + :param decay_rate: 平滑系数 + :param decay_steps: 时间步长 + :param is_stair: 是否平滑 + :param cls_name: 名称 + :return: + ''' validator.check_positive_int(decay_steps, 'decay_steps', cls_name) validator.check_positive_float(learning_rate, 'learning_rate', cls_name) validator.check_is_float(learning_rate, 'learning_rate', cls_name) @@ -52,6 +66,7 @@ def _check_inputs(learning_rate, decay_rate, decay_steps, is_stair, cls_name): class ExponentialDecayLR(LearningRateSchedule): + # 基于指数衰减函数计算学习率。 r""" Calculates learning rate base on exponential decay function. @@ -102,6 +117,13 @@ class ExponentialDecayLR(LearningRateSchedule): 0.09486833 """ def __init__(self, learning_rate, decay_rate, decay_steps, is_stair=False): + ''' + 初始化ExponentialDecayLR类，并使用_check_inputs方法检查参数 + :param learning_rate: 学习率 + :param decay_rate: 平滑系数 + :param decay_steps: 时间步长 + :param is_stair: 是否平滑 + ''' super(ExponentialDecayLR, self).__init__() _check_inputs(learning_rate, decay_rate, decay_steps, is_stair, self.cls_name) self.learning_rate = learning_rate @@ -112,13 +134,21 @@ class ExponentialDecayLR(LearningRateSchedule): self.cast = P.Cast() def construct(self, global_step): + ''' + 构建ExponentialDecayLR类 + :param global_step: 步骤 + :return: + ''' p = self.cast(global_step, mstype.float32) / self.decay_steps + # 如果是在梯度下降时，则使用Floor函数 if self.is_stair: p = P.Floor()(p) + # 返回指数衰减学习率 return self.learning_rate * self.pow(self.decay_rate, p) class NaturalExpDecayLR(LearningRateSchedule): + # 基于自然指数衰减函数计算学习率。 r""" Calculates learning rate base on natural exponential decay function. @@ -169,6 +199,13 @@ class NaturalExpDecayLR(LearningRateSchedule): 0.1 """ def __init__(self, learning_rate, decay_rate, decay_steps, is_stair=False): + ''' + 初始化一个NaturalExpDecayLR类，并使用_check_inputs方法检查参数 + :param learning_rate: 学习率 + :param decay_rate: 分段衰减率 + :param decay_steps: 时间步长 + :param is_stair: 是否变为折线图 + ''' super(NaturalExpDecayLR, self).__init__() _check_inputs(learning_rate, decay_rate, decay_steps, is_stair, self.cls_name) self.learning_rate = learning_rate @@ -180,13 +217,21 @@ class NaturalExpDecayLR(LearningRateSchedule): self.cast = P.Cast() def construct(self, global_step): + ''' + 构建NaturalExpDecayLR类 + :param global_step: 步骤 + :return: + ''' p = self.cast(global_step, mstype.float32) + # 如果是在梯度下降模式下，则将p乘以decay_steps，并将p除以decay_steps if self.is_stair: p = P.FloorDiv()(p, self.decay_steps) * self.decay_steps + # 返回learning_rate乘以pow(math_e, -decay_rate * p) return self.learning_rate * self.pow(self.math_e, -self.decay_rate * p) class InverseDecayLR(LearningRateSchedule): + # 基于逆时衰减函数计算学习率。 r""" Calculates learning rate base on inverse-time decay function. @@ -237,6 +282,13 @@ class InverseDecayLR(LearningRateSchedule): 0.1 """ def __init__(self, learning_rate, decay_rate, decay_steps, is_stair=False): + ''' + 初始化InverseDecayLR，并使用_check_inputs方法检查参数 + :param learning_rate: 学习率 + :param decay_rate: 平滑系数 + :param decay_steps: 时间步长 + :param is_stair: 是否平滑 + ''' super(InverseDecayLR, self).__init__() _check_inputs(learning_rate, decay_rate, decay_steps, is_stair, self.cls_name) self.learning_rate = learning_rate @@ -246,13 +298,21 @@ class InverseDecayLR(LearningRateSchedule): self.cast = P.Cast() def construct(self, global_step): + ''' + 计算每次迭代的学习率 + :param global_step: 所有epoch的迭代次数 + :return: 每次迭代的学习率 + ''' p = self.cast(global_step, mstype.float32) / self.decay_steps + # 如果是在梯度下降过程中，则将p转换为浮点数 if self.is_stair: p = P.Floor()(p) + # 返回学习率的除法结果，使用指数衰减 return self.learning_rate / (1 + self.decay_rate * p) class CosineDecayLR(LearningRateSchedule): + # 基于余弦衰减函数计算学习率。 r""" Calculates learning rate base on cosine decay function. @@ -294,30 +354,50 @@ class CosineDecayLR(LearningRateSchedule): 0.055 """ def __init__(self, min_lr, max_lr, decay_steps): + ''' + 初始化CosineDecayLR类，并检查参数 + :param min_lr: 最小学习率 + :param max_lr: 最大学习率 + :param decay_steps: 时间步数 + ''' super(CosineDecayLR, self).__init__() if not isinstance(min_lr, float): raise TypeError("min_lr must be float.") validator.check_non_negative_float(min_lr, "min_lr", self.cls_name) - validator.check_positive_float(max_lr, 'max_lr', self.cls_name) - validator.check_is_float(max_lr, 'max_lr', self.cls_name) + validator.check_positive_float(max_lr,'max_lr', self.cls_name) + validator.check_is_float(max_lr,'max_lr', self.cls_name) validator.check_positive_int(decay_steps, "decay_steps", self.cls_name) if min_lr >= max_lr: raise ValueError('`max_lr` should be greater than `min_lr`.') + # 将min_lr和max_lr赋值给变量min_lr和max_lr self.min_lr = min_lr self.max_lr = max_lr + # 将decay_steps赋值给变量decay_steps self.decay_steps = decay_steps + # 将math.pi赋值给变量math_pi self.math_pi = math.pi + # 将delta赋值给变量delta self.delta = 0.5 * (max_lr - min_lr) + # 创建一个Cos函数 self.cos = P.Cos() + # 创建一个Minimum函数 self.min = P.Minimum() + # 创建一个Cast函数 self.cast = P.Cast() def construct(self, global_step): + ''' + 构建CosineDecayLR + :param global_step: 总共的时间步数 + :return: + ''' p = self.cast(self.min(global_step, self.decay_steps), mstype.float32) + # 计算p的值，并将其转换为float32类型 return self.min_lr + self.delta * (1.0 + self.cos(self.math_pi * p / self.decay_steps)) class PolynomialDecayLR(LearningRateSchedule): + # 基于多项式衰减函数计算学习率。 r""" Calculates learning rate base on polynomial decay function. @@ -371,6 +451,14 @@ class PolynomialDecayLR(LearningRateSchedule): 0.07363961 """ def __init__(self, learning_rate, end_learning_rate, decay_steps, power, update_decay_steps=False): + ''' + 初始化PolynomialDecayLR类，并检查参数 + :param learning_rate: 学习率 + :param end_learning_rate: 终止学习率 + :param decay_steps: 时间步数 + :param power: 指数 + :param update_decay_steps: 是否更新指数步长 + ''' super(PolynomialDecayLR, self).__init__() validator.check_positive_float(learning_rate, 'learning_rate') validator.check_is_float(learning_rate, 'learning_rate') @@ -394,18 +482,30 @@ class PolynomialDecayLR(LearningRateSchedule): self.max = P.Maximum() def construct(self, global_step): + ''' + 构建PolynomialDecayLR类 + :param global_step: 步数 + :return: 学习率 + ''' tmp_global_step = P.Cast()(global_step, mstype.float32) + # 将训练步数转换为浮点数 tmp_decay_step = self.decay_steps + # 如果update_decay_steps为True，则tmp_decay_step乘以max(ceil(tmp_global_step / tmp_decay_step), 1) if self.update_decay_steps: tmp_decay_step = tmp_decay_step * self.max(self.ceil(tmp_global_step / tmp_decay_step), 1) + # 否则，tmp_global_step小于tmp_decay_step，则tmp_global_step等于tmp_decay_step else: tmp_global_step = self.min(tmp_global_step, tmp_decay_step) + # tmp_global_step / tmp_decay_step p = tmp_global_step / tmp_decay_step + # 将p的值转换为float32类型，并乘以diff_learning_rate，加上end_learning_rate lr = self.diff_learning_rate * self.pow(1.0 - p, self.power) + self.end_learning_rate + # 返回lr return lr class WarmUpLR(LearningRateSchedule): + # 预热学习率。 r""" Gets learning rate warming up. @@ -448,6 +548,11 @@ class WarmUpLR(LearningRateSchedule): 0.1 """ def __init__(self, learning_rate, warmup_steps): + ''' + 初始化WarmUpLR类，并检查输入参数 + :param learning_rate: 学习率 + :param warmup_steps: 过去warmup_steps步后的学习率 + ''' super(WarmUpLR, self).__init__() if not isinstance(learning_rate, float): raise TypeError("learning_rate must be float.") @@ -459,10 +564,15 @@ class WarmUpLR(LearningRateSchedule): self.cast = P.Cast() def construct(self, global_step): + ''' + 计算学习率 + :param global_step: 步数 + :return: 学习率 + ''' warmup_percent = self.cast(self.min(global_step, self.warmup_steps), mstype.float32)/ self.warmup_steps + # 返回预热学习率，乘以warmup_percent return self.learning_rate * warmup_percent - __all__ = [ 'ExponentialDecayLR', 'NaturalExpDecayLR',