Browse Source

!8725 add definition of scale for GPT

From: @alouhahahahaha
Reviewed-by: @guoqi1024,@c_34
Signed-off-by: @guoqi1024
tags/v1.1.0
mindspore-ci-bot Gitee 5 years ago
parent
commit
8da2a59764
1 changed files with 1 additions and 0 deletions
  1. +1
    -0
      model_zoo/official/nlp/gpt/src/gpt.py

+ 1
- 0
model_zoo/official/nlp/gpt/src/gpt.py View File

@@ -361,6 +361,7 @@ class Block(nn.Cell):
""" """
def __init__(self, config, layer_idx): def __init__(self, config, layer_idx):
super(Block, self).__init__() super(Block, self).__init__()
scale = 1.0
self.layernorm1 = LayerNorm((config.embedding_size,)).to_float(config.compute_dtype) self.layernorm1 = LayerNorm((config.embedding_size,)).to_float(config.compute_dtype)
self.attention = Attention(config, scale, layer_idx) self.attention = Attention(config, scale, layer_idx)
self.layernorm2 = LayerNorm((config.embedding_size,)).to_float(config.compute_dtype) self.layernorm2 = LayerNorm((config.embedding_size,)).to_float(config.compute_dtype)


Loading…
Cancel
Save