mindspore2022/tests/ut/python/parallel/test_allreduce_fusion.py

# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import pytest
import mindspore as ms
import mindspore.nn as nn
from mindspore import Tensor, context
from mindspore.common.api import _cell_graph_executor
from mindspore.nn import TrainOneStepCell, WithLossCell
from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits
from mindspore.nn.optim import Lamb
from mindspore.nn.optim.momentum import Momentum
from mindspore.ops import operations as P
from mindspore.parallel import _cost_model_context as cost_model_context
from mindspore.parallel._auto_parallel_context import auto_parallel_context
from mindspore.train import Model
from mindspore.context import ParallelMode
from tests.dataset_mock import MindData
context.set_context(mode=context.PYNATIVE_MODE)

class Net(nn.Cell):
    """Net definition"""
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Dense(128, 768, activation='relu')
        self.fc2 = nn.Dense(128, 768, activation='relu')
        self.fc3 = nn.Dense(128, 768, activation='relu')
        self.fc4 = nn.Dense(768, 768, activation='relu')
        self.relu4 = nn.ReLU()
        self.relu5 = nn.ReLU()
        self.transpose = P.Transpose()
        self.matmul1 = P.MatMul()
        self.matmul2 = P.MatMul()

    def construct(self, x):
        q = self.fc1(x)
        k = self.fc2(x)
        v = self.fc3(x)
        k = self.transpose(k, (1, 0))
        c = self.relu4(self.matmul1(q, k))
        s = self.relu5(self.matmul2(c, v))
        s = self.fc4(s)
        return s

class Dataset(MindData):
    def __init__(self, predict, label, length=3):
        super(Dataset, self).__init__(size=length)
        self.predict = predict
        self.label = label
        self.index = 0
        self.length = length

    def __iter__(self):
        return self

    def __next__(self):
        if self.index >= self.length:
            raise StopIteration
        self.index += 1
        return self.predict, self.label

    def reset(self):
        self.index = 0

class DenseNet1(nn.Cell):
    def __init__(self, has_bias=True, activation='relu'):
        super(DenseNet1, self).__init__()
        self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

    def construct(self, x):
        q = self.fc1(x)
        k = self.fc2(q)
        v = self.fc3(k)
        s = self.fc4(v)
        return s

class DenseNet2(nn.Cell):
    def __init__(self, has_bias=True, activation='relu'):
        super(DenseNet2, self).__init__()
        self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc5 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc6 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc7 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
        self.fc8 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

    def construct(self, x):
        q = self.fc1(x)
        k = self.fc2(q)
        v = self.fc3(k)
        s = self.fc4(v)
        t = self.fc5(s)
        u = self.fc6(t)
        w = self.fc7(u)
        z = self.fc8(w)
        return z

class DenseNet3(nn.Cell):
    def __init__(self, has_bias=True, activation='relu'):
        super(DenseNet3, self).__init__()
        self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

    def construct(self, x):
        q = self.fc1(x)
        return q

class SimpleDMLNet(nn.Cell):
    def __init__(self, net1, net2):
        super(SimpleDMLNet, self).__init__()
        self.backbone1 = net1
        self.backbone2 = net2

    def construct(self, x):
        x1 = self.backbone1(x)
        x2 = self.backbone2(x)
        return x1 + x2

def train_common(net):
    batch_size = 32
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2
    device_num = 4
    context.set_auto_parallel_context(device_num=device_num, parameter_broadcast=False)
    context.set_context(mode=context.GRAPH_MODE)

    predict = Tensor(np.ones([batch_size, 128]), dtype=ms.float32)
    label = Tensor(np.ones([batch_size]), dtype=ms.int32)
    dataset = Dataset(predict, label, 2)

    loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    model = Model(net, loss, opt)

    model.train(epoch_size, dataset, dataset_sink_mode=False)
    allreduce_fusion_dict = _cell_graph_executor._get_allreduce_fusion(model._train_network)
    print(allreduce_fusion_dict)
    return allreduce_fusion_dict

def test_allreduce_fusion_auto():
    """
    Feature: test_allreduce_fusion in auto mode
    Description: allreduce fusion in auto mode
    Expectation: success
    """
    comm_fusion_dict = {"allreduce": {"mode": "auto", "config": None}}
    context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, comm_fusion=comm_fusion_dict)
    net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))
    allreduce_fusion_dict = train_common(net)
    expect_dict = {'backbone2.fc8.weight': 1,
                   'backbone2.fc7.weight': 1,
                   'backbone2.fc6.weight': 1,
                   'backbone1.fc4.weight': 1,
                   'backbone1.fc3.weight': 1,
                   'backbone1.fc2.weight': 1,
                   'backbone2.fc5.weight': 1,
                   'backbone2.fc4.weight': 1,
                   'backbone2.fc3.weight': 1,
                   'backbone2.fc2.weight': 1,
                   'backbone2.fc1.weight': 1,
                   'backbone1.fc1.weight': 1}
    assert allreduce_fusion_dict == expect_dict

def test_allreduce_fusion_size():
    """
    Feature: test_allreduce_fusion in size mode
    Description: allreduce fusion in size mode
    Expectation: success
    """
    comm_fusion_dict = {"allreduce": {"mode": "size", "config": 32}}
    context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, comm_fusion=comm_fusion_dict)
    net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))
    allreduce_fusion_dict = train_common(net)
    expect_dict = {'backbone2.fc8.weight': 1,
                   'backbone2.fc7.weight': 1,
                   'backbone2.fc6.weight': 1,
                   'backbone1.fc4.weight': 1,
                   'backbone1.fc3.weight': 1,
                   'backbone1.fc2.weight': 1,
                   'backbone2.fc5.weight': 1,
                   'backbone2.fc4.weight': 1,
                   'backbone2.fc3.weight': 1,
                   'backbone2.fc2.weight': 1,
                   'backbone2.fc1.weight': 1,
                   'backbone1.fc1.weight': 1}
    assert allreduce_fusion_dict == expect_dict
    cost_model_context.reset_cost_model_context()
    comm_fusion = auto_parallel_context().get_comm_fusion()
    assert comm_fusion_dict == comm_fusion

def test_lamb_split_fusion_in_index():
    """
    Feature: test_allreduce_fusion in index mode
    Description: allreduce fusion in index mode
    Expectation: success
    """
    comm_fusion_dict = {"allreduce": {"mode": "index", "config": [2, 4, 6, 8]}}
    context.set_auto_parallel_context(parallel_mode="data_parallel", device_num=2, enable_parallel_optimizer=True,
                                      comm_fusion=comm_fusion_dict)
    inputs = Tensor(np.ones([32, 128]).astype(np.float32))
    label = Tensor(np.zeros([32, 768]).astype(np.float32))
    net = Net()
    net.set_train()
    loss = nn.SoftmaxCrossEntropyWithLogits()
    optimizer = Lamb(net.trainable_params(), learning_rate=0.1)

    net_with_loss = WithLossCell(net, loss)
    train_network = TrainOneStepCell(net_with_loss, optimizer)
    _cell_graph_executor.compile(train_network, inputs, label)
    context.reset_auto_parallel_context()

def test_allreduce_fusion_size_priority():
    """
    Feature: test priority of "enable_all_reduce_fusion" and "comm_fusion"
    Description: test priority of "enable_all_reduce_fusion" and "comm_fusion"
    Expectation: success
    """
    auto_parallel_context().set_enable_all_reduce_fusion(enable_all_reduce_fusion=False)
    comm_fusion_dict = {"allreduce": {"mode": "size", "config": 32}}
    context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, comm_fusion=comm_fusion_dict)
    net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))
    allreduce_fusion_dict = train_common(net)
    expect_dict = {}
    assert allreduce_fusion_dict == expect_dict
    auto_parallel_context().set_enable_all_reduce_fusion(enable_all_reduce_fusion=True)
    allreduce_fusion_dict = train_common(net)
    expect_dict = {'backbone2.fc8.weight': 1,
                   'backbone2.fc7.weight': 1,
                   'backbone2.fc6.weight': 1,
                   'backbone1.fc4.weight': 1,
                   'backbone1.fc3.weight': 1,
                   'backbone1.fc2.weight': 1,
                   'backbone2.fc5.weight': 1,
                   'backbone2.fc4.weight': 1,
                   'backbone2.fc3.weight': 1,
                   'backbone2.fc2.weight': 1,
                   'backbone2.fc1.weight': 1,
                   'backbone1.fc1.weight': 1}
    assert allreduce_fusion_dict == expect_dict

def test_allreduce_fusion_size_one_tensor():
    """
    Feature: test_allreduce_fusion in size mode with one tensor
    Description: test_allreduce_fusion in size mode with one tensor
    Expectation: success
    """
    comm_fusion_dict = {"allreduce": {"mode": "size", "config": 32}}
    context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, comm_fusion=comm_fusion_dict)
    net = DenseNet3(has_bias=False, activation=None)
    allreduce_fusion_dict = train_common(net)
    expect_dict = {'fc1.weight': 1}
    assert allreduce_fusion_dict == expect_dict

def test_fusion_invalid_value_failed():
    """
    Feature: test_allreduce_fusion with invalid value
    Description: test_allreduce_fusion with invalid value
    Expectation: throw TypeError
    """
    with pytest.raises(TypeError):
        comm_fusion_dict = {"allreduce": {"mode": "size", "config": "30.12"}}
        context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, comm_fusion=comm_fusion_dict)

def test_enable_invalid_value_failed():
    """
    Feature: enable_all_reduce_fusion with invalid value
    Description: enable_all_reduce_fusion with invalid value
    Expectation: throw TypeError
    """
    with pytest.raises(TypeError):
        auto_parallel_context().set_enable_all_reduce_fusion(enable_all_reduce_fusion="fusion")