删除 mix_module 和 torch_paddle_driver 的内容

fastNLP/core/__init__.py

@@ -63,7 +63,6 @@ __all__ = [
     "PaddleFleetDriver",
     "JittorSingleDriver",
     "JittorMPIDriver",
-    "TorchPaddleDriver",
 
     # log
     "logger",

fastNLP/core/drivers/__init__.py

@@ -9,7 +9,6 @@ __all__ = [
     "JittorDriver",
     "JittorSingleDriver",
     "JittorMPIDriver",
-    "TorchPaddleDriver",
     'torch_seed_everything',
     'paddle_seed_everything',
     'optimizer_state_to_device'
@@ -18,7 +17,6 @@ __all__ = [
 from .torch_driver import TorchDriver, TorchSingleDriver, TorchDDPDriver, torch_seed_everything, optimizer_state_to_device
 from .jittor_driver import JittorDriver, JittorMPIDriver, JittorSingleDriver
 from .paddle_driver import PaddleDriver, PaddleFleetDriver, PaddleSingleDriver, paddle_seed_everything
-from .torch_paddle_driver import TorchPaddleDriver
 from .driver import Driver
 
 

fastNLP/core/drivers/torch_paddle_driver/__init__.py

@@ -1,5 +0,0 @@
-__all__ = [
-    "TorchPaddleDriver",
-]
-
-from .torch_paddle_driver import TorchPaddleDriver

fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py

@@ -1,193 +0,0 @@
-from typing import Optional, Dict, Union, Callable, Tuple
-
-from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH
-from fastNLP.core.utils.utils import _get_fun_msg
-
-
-if _NEED_IMPORT_PADDLE:
-    import paddle
-    from paddle.io import DataLoader as PaddleDataLoader
-    from paddle.optimizer import Optimizer as PaddleOptimizer
-
-if _NEED_IMPORT_TORCH:
-    import torch
-    from torch.utils.data import DataLoader as TorchDataLoader
-    from torch.optim import Optimizer as TorchOptimizer
-
-from fastNLP.core.drivers.driver import Driver
-from fastNLP.envs.distributed import rank_zero_call
-from fastNLP.core.utils.utils import auto_param_call, apply_to_collection
-from fastNLP.core.log.logger import logger
-from fastNLP.modules.mix_modules.mix_module import MixModule
-
-
-__all__ = [
-    "TorchPaddleDriver",
-]
-
-class TorchPaddleDriver(Driver):
-    """
-    针对torch和paddle混合模型的driver
-    由于是两种不同的框架不方便实现多卡，暂时先实现CPU和GPU单卡的功能
-    """
-    def __init__(self, model, device: Optional[str] = None, **kwargs):
-        super(TorchPaddleDriver, self).__init__(model)
-
-        self.model_device = device
-        self.torch_non_blocking = kwargs.get("torch_non_blocking", None)
-        self.paddle_blocking = kwargs.get("paddle_blocking", None)
-
-        self._data_device = kwargs.get("_data_device", None)
-        if isinstance(self._data_device, int):
-            # 将data_device设置为cuda:x的字符串形式
-            if self._data_device < 0:
-                raise ValueError("Parameter `_data_device` can not be smaller than 0.")
-            _could_use_device_num = paddle.device.cuda.device_count()
-            if self._data_device >= _could_use_device_num:
-                raise ValueError("The gpu device that parameter `device` specifies is not existed.")
-            self._data_device = f"cuda:{self._data_device}"
-        elif self._data_device is not None:
-            raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.")
-
-    def setup(self):
-        if self.model_device is not None:
-            paddle.device.set_device(self.model_device.replace("cuda", "gpu"))
-            self.model.to(self.model_device)
-
-    @staticmethod
-    def check_dataloader_legality(dataloader, dataloader_name, is_train: bool = False):
-        if is_train:
-            if not isinstance(dataloader, (TorchDataLoader, PaddleDataLoader)):
-                raise ValueError(f"Parameter `{dataloader_name}` should be 'torch.util.data.DataLoader' or `paddle.io.dataloader` type, not {type(dataloader)}.")
-        else:
-            if not isinstance(dataloader, Dict):
-                raise ValueError(f"Parameter `{dataloader_name}` should be 'Dict' type, not {type(dataloader)}.")
-            else:
-                for each_dataloader in dataloader.values():
-                    if not isinstance(each_dataloader, (TorchDataLoader, PaddleDataLoader)):
-                        raise ValueError(f"Each dataloader of parameter `{dataloader_name}` should be "
-                                         f"'torch.util.data.DataLoader' or `paddle.io.dataloader` "
-                                         f"type, not {type(each_dataloader)}.")
-
-    @staticmethod
-    def _check_optimizer_legality(optimizers):
-        for each_optimizer in optimizers:
-            if not isinstance(each_optimizer, (TorchOptimizer, PaddleOptimizer)):
-                raise ValueError(f"Each optimizers of parameter `optimizers` should be "
-                                 f"'torch.optim.Optimizer' or 'paddle.optimizers.Optimizer' type, "
-                                 f"not {type(each_optimizer)}.")
-
-    def step(self):
-        for optimizer in self.optimizers:
-            optimizer.step()
-
-    def backward(self, loss):
-        loss.backward()
-
-    def zero_grad(self):
-        for optimizer in self.optimizers:
-            if isinstance(optimizer, TorchOptimizer):
-                optimizer.zero_grad()
-            elif isinstance(optimizer, PaddleOptimizer):
-                optimizer.clear_grad()
-            else:
-                raise ValueError("Unknown optimizers type.")
-
-    def model_call(self, batch, fn: Callable, signature_fn: Optional[Callable]) -> Dict:
-        if isinstance(batch, Dict) and not self.wo_auto_param_call:
-            return auto_param_call(fn, batch, signature_fn=signature_fn)
-        else:
-            return fn(batch)
-
-    def get_model_call_fn(self, fn: str) -> Tuple:
-        if hasattr(self.model, fn):
-            fn = getattr(self.model, fn)
-            if not callable(fn):
-                raise RuntimeError(f"The `{fn}` attribute is not `Callable`.")
-            logger.debug(f'Use {_get_fun_msg(fn, with_fp=False)}...')
-            return fn, None
-        elif fn in {"train_step", "evaluate_step"}:
-            logger.debug(f'Use {_get_fun_msg(self.model.forward, with_fp=False)}...')
-            return self.model, self.model.forward
-        else:
-            raise RuntimeError(f"There is no `{fn}` method in your {type(self.model)}.")
-
-    def predict_step(self, batch):
-        if isinstance(batch, Dict):
-            return auto_param_call(self._predict_step, batch)
-        else:
-            return self._predict_step(batch)
-
-    @rank_zero_call
-    def save_model(self, filepath: str, only_state_dict: bool = True, model_save_fn: Optional[Callable] = None):
-        r"""
-        暂时不提供保存整个模型的方法
-        """
-        if only_state_dict == False:
-            logger.warn("TorchPaddleModule only support saving state dicts now.")
-        if model_save_fn is not None:
-            model_save_fn(filepath)
-        else:
-            model = self.unwrap_model()
-            self.move_model_to_device(model, "cpu")
-            self.model.save(filepath)
-            self.move_model_to_device(model, self.model_device)
-
-    def load_model(self, filepath: str):
-        """
-        加载模型的加载函数；
-
-        :param filepath: 保存文件的文件位置（需要包括文件名）；
-        :return:
-        """
-        return self.model.load(filepath)
-
-    def save(self):
-        ...
-
-    def load(self):
-        ...
-
-    @staticmethod
-    def move_model_to_device(model: MixModule, device: str):
-        if device is not None:
-            model.to(device)
-
-    def unwrap_model(self):
-        return self.model
-
-    @staticmethod
-    def tensor_to_numeric(tensor):
-        if tensor is None:
-            return None
-
-        def _translate(_data):
-            return _data.tolist()
-
-        return apply_to_collection(
-            data=tensor,
-            dtype=(paddle.Tensor, torch.Tensor),
-            function=_translate
-        )
-
-    def set_model_mode(self, mode: str):
-        assert mode in {"train", "eval"}
-        getattr(self.model, mode)()
-
-    def get_model_device(self):
-        return self.model_device
-
-    @property
-    def data_device(self):
-        if self.model_device is not None:
-            return self.model_device
-        else:
-            return self._data_device
-
-    def set_model_mode(self, mode: str):
-        assert mode in {"train", "eval"}
-        getattr(self.model, mode)()
-
-    def set_sampler_epoch(self, dataloader: Union['TorchDataLoader', 'PaddleDataLoader'], cur_epoch_idx):
-        # 保证 ddp 训练时的 shuffle=True 时的正确性，因为需要保证每一个进程上的 sampler 的shuffle 的随机数种子是一样的；
-        return dataloader

fastNLP/core/drivers/torch_paddle_driver/utils.py

@@ -1,4 +0,0 @@
-from fastNLP.envs.imports import _NEED_IMPORT_PADDLE
-
-if _NEED_IMPORT_PADDLE:
-    pass

fastNLP/core/utils/__init__.py

@@ -11,7 +11,6 @@ __all__ = [
     'is_in_fnlp_paddle_dist',
     'is_in_paddle_launch_dist',
     'f_rich_progress',
-    'torch_paddle_move_data_to_device',
     'torch_move_data_to_device',
     'get_fn_arg_names',
     'auto_param_call',
@@ -32,7 +31,6 @@ from .jittor_utils import is_jittor_dataset, jittor_collate_wraps
 from .paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \
     is_in_fnlp_paddle_dist, is_in_paddle_launch_dist
 from .rich_progress import f_rich_progress
-from .torch_paddle_utils import torch_paddle_move_data_to_device
 from .torch_utils import torch_move_data_to_device
 from .utils import *
 

fastNLP/core/utils/torch_paddle_utils.py

@@ -1,49 +0,0 @@
-from typing import Any, Optional
-
-from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH
-
-if _NEED_IMPORT_PADDLE:
-    import paddle
-
-if _NEED_IMPORT_TORCH:
-    import torch
-
-__all__ = [
-    "torch_paddle_move_data_to_device",
-]
-
-from .utils import apply_to_collection
-from .paddle_utils import paddle_to
-
-
-def torch_paddle_move_data_to_device(batch: Any, device: Optional[str] = None, non_blocking: Optional[bool] = True, 
-                                    data_device: Optional[str] = None) -> Any:
-    
-    r"""
-    将数据集合传输到给定设备。只有paddle.Tensor和torch.Tensor对象会被传输到设备中，其余保持不变
-
-    :param batch:
-    :param device:
-    :param non_blocking:
-    :param data_device:
-    :return: 相同的集合，但所有包含的张量都驻留在新设备上；
-    """
-
-    if device is None:
-        if data_device is not None:
-            device = data_device
-        else:
-            return batch
-
-    torch_device = device.replace("gpu", "cuda")
-    paddle_device = device.replace("cuda", "gpu")
-
-    def batch_to(data: Any) -> Any:
-        if isinstance(data, torch.Tensor):
-            data = data.to(torch_device, non_blocking=non_blocking)
-        elif isinstance(data, paddle.Tensor):
-            data = paddle_to(data, paddle_device)
-        
-        return data
-
-    return apply_to_collection(batch, dtype=(paddle.Tensor, torch.Tensor), function=batch_to)

fastNLP/modules/__init__.py

@@ -1,9 +0,0 @@
-__all__ = [
-    "MixModule",
-    "torch2paddle",
-    "paddle2torch",
-    "torch2jittor",
-    "jittor2torch",
-]
-
-from .mix_modules import MixModule, torch2paddle, paddle2torch, torch2jittor, jittor2torch

fastNLP/modules/mix_modules/__init__.py

@@ -1,10 +0,0 @@
-__all__ = [
-    "MixModule",
-    "torch2paddle",
-    "paddle2torch",
-    "torch2jittor",
-    "jittor2torch",
-]
-
-from .mix_module import MixModule
-from .utils import *

fastNLP/modules/mix_modules/mix_module.py

@@ -1,310 +0,0 @@
-import os
-import io
-import pickle
-from typing import Dict
-from collections import OrderedDict
-
-import numpy as np
-
-from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH
-from fastNLP.core.utils.paddle_utils import paddle_to
-
-if _NEED_IMPORT_PADDLE:
-    import paddle
-    from paddle.nn import Layer as PaddleLayer
-
-if _NEED_IMPORT_TORCH:
-    import torch
-    from torch.nn import Module as TorchModule, Parameter as TorchParameter
-
-if _NEED_IMPORT_JITTOR:
-    import jittor
-
-
-__all__ = [
-    "MixModule",
-]
-
-class MixModule:
-    """
-    TODO: 支持不同的混合方式；添加state_dict的支持；如果参数里有List of Tensors该怎么处理；
-        是否需要仿照Module那样在初始化的时候给各种模型分类
-    可以同时使用Torch和Paddle框架的混合模型
-    """
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def __call__(self, *args, **kwargs):
-        return self.forward(*args, **kwargs)
-
-    def named_parameters(self, prefix='', recurse: bool=True, backend=None):
-        """
-        返回模型的名字和参数
-
-        :param prefix: 输出时在参数名前加上的前缀
-        :param recurse: 是否递归地输出参数
-        :param backend: `backend`=`None`时，将所有模型和张量的参数返回；
-                        `backend`=`torch`时，返回`torch`的参数；
-                        `backend`=`paddle`时，返回`paddle`的参数。
-        """
-        if backend is None:
-            generator = self.attributes(TorchModule, TorchParameter, PaddleLayer)
-        elif backend == "torch":
-            generator = self.attributes(TorchModule, TorchParameter)
-        elif backend == "paddle":
-            generator = self.attributes(PaddleLayer)
-        else:
-            raise ValueError("Unknown backend parameter.")
-
-        for name, value in generator:
-            name = prefix + ('.' if prefix else '') + name
-            if isinstance(value, TorchParameter):
-                # 非Module/Layer类型，直接输出名字和值
-                yield name, value
-            elif recurse:
-                # 递归地调用named_parameters
-                for name_r, value_r in value.named_parameters(name, recurse):
-                    yield name_r, value_r
-
-    def parameters(self, recurse: bool = True, backend: str = None):
-        """
-        返回模型的参数
-
-        :param recurse:
-        :param backend: `backend`=`None`时，将所有模型和张量的参数返回；
-                        `backend`=`torch`时，返回`torch`的参数；
-                        `backend`=`paddle`时，返回`paddle`的参数。
-        """
-        for name, value in self.named_parameters(recurse=recurse, backend=backend):
-            yield value
-    
-    def forward(self, *args, **kwargs):
-        raise NotImplementedError
-
-    def train_step(self, batch):
-        raise NotImplementedError
-
-    def test_step(self, batch):
-        raise NotImplementedError
-
-    def evaluate_step(self, batch):
-        raise NotImplementedError
-
-    def train(self):
-        for name, value in self.attributes(TorchModule, PaddleLayer):
-            value.train()
-
-    def eval(self):
-        for name, value in self.attributes(TorchModule, PaddleLayer):
-            value.eval()
-
-    def to(self, device):
-        """
-        :param device: 设备名
-        """
-        # 有jittor的话 warning
-        if device == "cpu":
-            paddle_device = device
-        elif device.startswith("cuda"):
-            paddle_device = device.replace("cuda", "gpu")
-        elif device.startswith("gpu"):
-            paddle_device = device
-            device = device.replace("gpu", "cuda")
-        else:
-            raise ValueError("Device value error")
-
-        for name, value in self.attributes(TorchModule):
-            # torch的to函数不影响Tensor
-            vars(self)[name] = value.to(device)
-        for name, value in self.attributes(TorchParameter):
-            # Parameter在经过to函数后会变成Tensor类型
-            vars(self)[name] = TorchParameter(value.to(device), requires_grad=value.requires_grad)
-
-        for name, value in self.attributes(PaddleLayer):
-            vars(self)[name] = value.to(paddle_device)
-        for name, value in self.attributes(paddle.Tensor):
-        # paddle的to函数会影响到Tensor
-            vars(self)[name] = paddle_to(value, paddle_device)
-
-        return self
-
-    def state_dict(self, backend: str = None) -> Dict:
-        """
-        返回模型的state_dict。
-
-        .. note:: torch的destination参数会在将来删除，因此不提供destination参数
-
-        :param backend: `backend`=`None`时，将所有模型和张量的state dict返回；
-                        `backend`=`torch`时，返回`torch`的state dict；
-                        `backend`=`paddle`时，返回`paddle`的state dict。
-        """
-        if backend is None:
-            generator = self.attributes(TorchModule, TorchParameter, PaddleLayer)
-        elif backend == "torch":
-            generator = self.attributes(TorchModule, TorchParameter)
-        elif backend == "paddle":
-            generator = self.attributes(PaddleLayer)
-        else:
-            raise ValueError(f"Unknown backend {backend}.")
-
-        destination = OrderedDict()
-
-        for name, value in generator:
-            if value is None:
-                continue
-            if isinstance(value, TorchParameter):
-                destination[name] = value
-            else:
-                # 不同框架state_dict函数的参数名和顺序不同
-                if isinstance(value, PaddleLayer):
-                    kwargs = {
-                        "structured_name_prefix": name + ".",
-                    }
-                elif isinstance(value, TorchModule):
-                    kwargs = {
-                        "prefix": name + ".",
-                    }
-                else:
-                    raise ValueError(f"Unknown item type {type(value)}")
-                destination.update(value.state_dict(**kwargs))
-
-        return destination
-
-    def save_state_dict_to_file(self, path: str):
-        """
-        保存模型的state dict到path
-        """
-        # TODO 设备限制
-        filename = os.path.basename(path)
-        if filename == "":
-            raise ValueError("Received empty filename.")
-        dirname = os.path.dirname(path)
-        if dirname and not os.path.exists(dirname):
-            os.makedirs(dirname)
-        protocol = 4
-
-        saved = {}
-        paddle_dict = self.state_dict(backend="paddle")
-        torch_dict = self.state_dict(backend="torch")
-        # 保存paddle部分
-        # 调用paddle保存时的处理函数
-        paddle_saved_obj = paddle.framework.io._build_saved_state_dict(paddle_dict)
-        paddle_saved_obj = paddle.fluid.io._unpack_saved_dict(paddle_saved_obj, protocol)
-        # 将返回的dict保存
-        saved["paddle"] = paddle_saved_obj
-
-        # 保存torch部分
-        buffer = io.BytesIO()
-        torch.save(torch_dict, buffer)
-        saved["torch"] = buffer.getvalue()
-
-        # 保存
-        with open(path, "wb") as f:
-            pickle.dump(saved, f, protocol)
-
-    def load_state_dict_from_file(self, path: str):
-        """
-        从 `path` 中加载保存的state dict
-        """
-        state_dict = {}
-        with open(path, "rb") as f:
-            loaded = pickle.load(f)
-        # 加载paddle的数据
-        paddle_loaded_obj = loaded["paddle"]
-        paddle_load_result = paddle.fluid.io._pack_loaded_dict(paddle_loaded_obj)
-        if "StructuredToParameterName@@" in paddle_load_result:
-            for key in paddle_load_result["StructuredToParameterName@@"]:
-                if isinstance(paddle_load_result[key], np.ndarray):
-                    paddle_load_result[key] = paddle.to_tensor(paddle_load_result[key])
-        state_dict.update(paddle_load_result)
-        # 加载torch的数据
-        torch_loaded_obj = loaded["torch"]
-        torch_bytes = io.BytesIO(torch_loaded_obj)
-        torch_load_result = torch.load(torch_bytes)
-        state_dict.update(torch_load_result)
-
-        self.load_state_dict(state_dict)
-
-    def load_state_dict(self, state_dict):
-        """
-        从state dict中加载数据
-        """
-        missing_keys = []
-        unexpected_keys = []
-        error_msgs = []
-        new_state = {}
-
-        local_state = self.state_dict()
-
-        # 对字典内容按前缀进行归类
-        for key, value in state_dict.items():
-            splited = key.split(".", 1)
-            if len(splited) == 1:
-                # 没有前缀，实际上只有torch.nn.Parameter会进入这种情况
-                new_state[key] = value
-            else:
-                prefix, name = splited
-                if prefix not in new_state:
-                    new_state[prefix] = {}
-                new_state[prefix][name] = value
-
-        for key, param in self.attributes(TorchModule, TorchParameter, PaddleLayer):
-            if key in new_state:
-                # 在传入的字典中找到了对应的值
-                input_param = new_state[key]
-                if not isinstance(input_param, dict):
-                    # 且不是字典，即上述没有前缀的情况
-                    # 按照torch.nn.Module._load_from_state_dict进行赋值
-                    if not torch.overrides.is_tensor_like(input_param):
-                        error_msgs.append('While copying the parameter named "{}", '
-                                        'expected torch.Tensor or Tensor-like object from checkpoint but '
-                                        'received {}'
-                                        .format(key, type(input_param)))
-                        continue
-
-                    # This is used to avoid copying uninitialized parameters into
-                    # non-lazy modules, since they dont have the hook to do the checks
-                    # in such case, it will error when accessing the .shape attribute.
-                    is_param_lazy = torch.nn.parameter.is_lazy(param)
-                    # Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+
-                    if not is_param_lazy and len(param.shape) == 0 and len(input_param.shape) == 1:
-                        input_param = input_param[0]
-
-                    if not is_param_lazy and input_param.shape != param.shape:
-                        # local shape should match the one in checkpoint
-                        error_msgs.append('size mismatch for {}: copying a param with shape {} from checkpoint, '
-                                        'the shape in current model is {}.'
-                                        .format(key, input_param.shape, param.shape))
-                        continue
-                    try:
-                        with torch.no_grad():
-                            param.copy_(input_param)
-                    except Exception as ex:
-                        error_msgs.append('While copying the parameter named "{}", '
-                                        'whose dimensions in the model are {} and '
-                                        'whose dimensions in the checkpoint are {}, '
-                                        'an exception occurred : {}.'
-                                        .format(key, param.size(), input_param.size(), ex.args))
-                else:
-                    # 否则在子模块中
-                    if isinstance(param, TorchModule):
-                        # torch模块
-                        # 由于paddle没有提供类似strict的参数，因此也不对torch作要求
-                        param.load_state_dict(input_param, strict=False)
-                    elif isinstance(param, PaddleLayer):
-                        # paddle模块
-                        param.load_dict(input_param)
-            else:
-                missing_keys.append(key)
-
-        if len(error_msgs) > 0:
-            raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
-                               self.__class__.__name__, "\n\t".join(error_msgs)))
-
-    def attributes(self, *types):
-        """
-        查找对应类型的成员
-        """
-        for name, value in vars(self).items():
-            if isinstance(value, types):
-                yield name, value

fastNLP/modules/mix_modules/utils.py

@@ -1,233 +0,0 @@
-import warnings
-import os
-from typing import Any, Optional, Union
-
-import numpy as np
-
-from fastNLP.core.utils.utils import apply_to_collection
-from fastNLP.core.utils.paddle_utils import paddle_to
-from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_TORCH, _NEED_IMPORT_PADDLE
-
-if _NEED_IMPORT_PADDLE:
-    import paddle
-
-if _NEED_IMPORT_JITTOR:
-    import jittor
-
-if _NEED_IMPORT_TORCH:
-    import torch
-
-__all__ = [
-    "paddle2torch",
-    "torch2paddle",
-    "jittor2torch",
-    "torch2jittor",
-]
-
-def _paddle2torch(paddle_tensor: 'paddle.Tensor', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor':
-    """
-    将paddle tensor转换为torch tensor，并且能够保留梯度进行反向传播
-    :param paddle_tensor: 要转换的paddle张量
-    :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 转换后的torch张量
-    """
-    no_gradient = paddle_tensor.stop_gradient if no_gradient is None else no_gradient
-    paddle_numpy = paddle_tensor.numpy()
-    if not np.issubdtype(paddle_numpy.dtype, np.inexact):
-        no_gradient = True
-
-    if target_device is None:
-        if paddle_tensor.place.is_gpu_place():
-            # paddlepaddle有两种Place，对应不同的device id获取方式
-            if hasattr(paddle_tensor.place, "gpu_device_id"):
-                # paddle.fluid.core_avx.Place
-                # 在gpu环境下创建张量的话，张量的place是这一类型
-                target_device = f"cuda:{paddle_tensor.place.gpu_device_id()}"
-            else:
-                # paddle.CUDAPlace
-                target_device = f"cuda:{paddle_tensor.place.get_device_id()}"
-        else:
-            # TODO: 可能需要支持xpu等设备
-            target_device = "cpu"
-
-    if not no_gradient:
-        # 保持梯度，并保持反向传播
-        # torch.tensor会保留numpy数组的类型
-        torch_tensor = torch.tensor(paddle_numpy, requires_grad=True, device=target_device)
-        hook = torch_tensor.register_hook(
-            lambda grad: paddle.autograd.backward(paddle_tensor, paddle.to_tensor(grad.cpu().numpy()))
-        )
-    else:
-        # 不保留梯度
-        torch_tensor = torch.tensor(paddle_numpy, requires_grad=False, device=target_device)
-
-    return torch_tensor
-
-
-def _torch2paddle(torch_tensor: 'torch.Tensor', target_device: str = None, no_gradient: bool = None) -> 'paddle.Tensor':
-    """
-    将torch tensor转换为paddle tensor，并且能够保留梯度进行反向传播。
-    :param torch_tensor: 要转换的torch张量
-    :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 转换后的paddle张量
-    """
-    no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient
-    if target_device is None:
-        if torch_tensor.is_cuda:
-            target_device = f"gpu:{torch_tensor.device.index}"
-        else:
-            target_device = "cpu"
-
-    if not no_gradient:
-        # 保持梯度并保持反向传播
-        # paddle的stop_gradient和torch的requires_grad表现是相反的
-        paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=False)
-        hook = paddle_tensor.register_hook(
-            lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy()))
-        )
-    else:
-        paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=True)
-
-    paddle_tensor = paddle_to(paddle_tensor, target_device)
-
-    return paddle_tensor
-
-
-def _jittor2torch(jittor_var: 'jittor.Var', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor':
-    """
-    将jittor Var转换为torch tensor，并且能够保留梯度进行反向传播
-    :param jittor_var: 要转换的jittor变量
-    :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，根据jittor.flags.use_cuda决定。
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 转换后的torch张量
-    """
-    # TODO: warning：无法保留梯度
-    # jittor的grad可以通过callback进行传递
-    # 如果outputs有_grad键，可以实现求导
-    no_gradient = not jittor_var.requires_grad if no_gradient is None else no_gradient
-    if no_gradient == False:
-        warnings.warn("The result tensor will not keep gradients due to differences between jittor and pytorch.")
-    jittor_numpy = jittor_var.numpy()
-    if not np.issubdtype(jittor_numpy.dtype, np.inexact):
-        no_gradient = True
-
-    if target_device is None:
-        # jittor的设备分配是自动的
-        # 根据use_cuda判断
-        if jittor.flags.use_cuda:
-            target_device = "cuda:0"
-        else:
-            target_device = "cpu"
-
-    torch_tensor = torch.tensor(jittor_numpy, requires_grad=not no_gradient, device=target_device)
-
-    return torch_tensor
-
-
-def _torch2jittor(torch_tensor: 'torch.Tensor', no_gradient: bool = None) -> 'jittor.Var':
-    """
-    将torch tensor转换为jittor Var，并且能够保留梯度进行反向传播
-    :param torch_tensor: 要转换的torch张量
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 转换后的jittor变量
-    """
-    no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient
-
-    if not no_gradient:
-        # 保持梯度并保持反向传播
-        jittor_var = jittor.Var(torch_tensor.detach().numpy())
-        jittor_var.requires_grad = True
-        hook = jittor_var.register_hook(
-            lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy()))
-        )
-    else:
-        jittor_var = jittor.Var(torch_tensor.detach().numpy())
-        jittor_var.requires_grad = False
-
-    return jittor_var
-
-
-def torch2paddle(torch_in: Any, target_device: str = None, no_gradient: bool = None) -> Any:
-    """
-    递归地将输入中包含的torch张量转换为paddle张量
-
-    :param torch_in: 要转换的包含torch.Tensor类型的变量
-    :param target_device: 是否将转换后的张量迁移到特定设备上，
-                          输入为`None`时，和输入的张量相同，
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 将所有torch.Tensor转换为paddle.Tensor的张量           
-    """
-
-    return apply_to_collection(
-        torch_in,
-        dtype=torch.Tensor,
-        function=_torch2paddle,
-        target_device=target_device,
-        no_gradient=no_gradient,
-    )
-
-
-def paddle2torch(paddle_in: Any, target_device: str = None, no_gradient: bool = None) -> Any:
-    """
-    递归地将输入中包含的paddle张量转换为torch张量
-
-    :param torch_in: 要转换的包含paddle.Tensor类型的变量
-    :param target_device: 是否将转换后的张量迁移到特定设备上，
-                          输入为`None`时，和输入的张量相同，
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 将所有paddle.Tensor转换为torch.Tensor后的变量          
-    """
-
-    return apply_to_collection(
-        paddle_in,
-        dtype=paddle.Tensor,
-        function=_paddle2torch,
-        target_device=target_device,
-        no_gradient=no_gradient,
-    )
-
-
-def jittor2torch(jittor_in: Any, target_device: str = None, no_gradient: bool = None) -> Any:
-    """
-    递归地将输入中包含的jittor变量转换为torch张量
-
-    :param jittor_in: 要转换的jittor变量
-    :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，默认为cuda:0。
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 转换后的torch张量
-    """
-
-    return apply_to_collection(
-        jittor_in,
-        dtype=jittor.Var,
-        function=_jittor2torch,
-        target_device=target_device,
-        no_gradient=no_gradient,
-    )
-
-
-def torch2jittor(torch_in: Any, no_gradient: bool = None) -> Any:
-    """
-    递归地将输入中包含的torch张量转换为jittor变量
-
-    :param torch_tensor: 要转换的torch张量
-    :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致；
-                        为`True`时，全部不保留梯度；为`False`时，全部保留梯度。
-    :return: 转换后的jittor变量
-    """
-    
-    return apply_to_collection(
-        torch_in,
-        dtype=torch.Tensor,
-        function=_torch2jittor,
-        no_gradient=no_gradient,
-    )

tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py

@@ -1,122 +0,0 @@
-import pytest
-
-from fastNLP.modules.mix_modules.mix_module import MixModule
-from fastNLP.core.drivers.torch_paddle_driver.torch_paddle_driver import TorchPaddleDriver
-from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle
-
-import torch
-import paddle
-from paddle.io import Dataset, DataLoader
-import numpy as np
-
-############################################################################
-#
-# 测试在MNIST数据集上的表现
-#
-############################################################################
-
-class MNISTDataset(Dataset):
-    def __init__(self, dataset):
-
-        self.dataset = [
-            (
-                np.array(img).astype('float32').reshape(-1),
-                label
-            ) for img, label in dataset
-        ]
-
-    def __getitem__(self, idx):
-        return self.dataset[idx]
-
-    def __len__(self):
-        return len(self.dataset)
-
-class MixMNISTModel(MixModule):
-    def __init__(self):
-        super(MixMNISTModel, self).__init__()
-
-        self.fc1 = paddle.nn.Linear(784, 64)
-        self.fc2 = paddle.nn.Linear(64, 32)
-        self.fc3 = torch.nn.Linear(32, 10)
-        self.fc4 = torch.nn.Linear(10, 10)
-
-    def forward(self, x):
-
-        paddle_out = self.fc1(x)
-        paddle_out = self.fc2(paddle_out)
-        torch_in = paddle2torch(paddle_out)
-        torch_out = self.fc3(torch_in)
-        torch_out = self.fc4(torch_out)
-
-        return torch_out
-
-    def train_step(self, x):
-        return self.forward(x)
-
-    def test_step(self, x):
-        return self.forward(x)
-
-@pytest.mark.torchpaddle
-class TestMNIST:
-
-    @classmethod
-    def setup_class(self):
-
-        self.train_dataset = paddle.vision.datasets.MNIST(mode='train')
-        self.test_dataset = paddle.vision.datasets.MNIST(mode='test')
-        self.train_dataset = MNISTDataset(self.train_dataset)
-
-        self.lr = 0.0003
-        self.epochs = 20
-
-        self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True)
-
-    def setup_method(self):
-        
-        model = MixMNISTModel()
-        self.torch_loss_func = torch.nn.CrossEntropyLoss()
-
-        torch_opt = torch.optim.Adam(model.parameters(backend="torch"), self.lr)
-        paddle_opt = paddle.optimizer.Adam(parameters=model.parameters(backend="paddle"), learning_rate=self.lr)
-
-        self.driver = TorchPaddleDriver(model=model, device="cuda:0")
-        self.driver.set_optimizers([torch_opt, paddle_opt])
-
-    def test_case1(self):
-
-        epochs = 20
-
-        self.driver.setup()
-        self.driver.zero_grad()
-        # 开始训练
-        current_epoch_idx = 0
-        while current_epoch_idx < epochs:
-            epoch_loss, batch = 0, 0
-            self.driver.set_model_mode("train")
-            self.driver.set_sampler_epoch(self.dataloader, current_epoch_idx)
-            for batch, (img, label) in enumerate(self.dataloader):
-                img = paddle.to_tensor(img).cuda()
-                torch_out = self.driver.train_step(img)
-                label = torch.from_numpy(label.numpy()).reshape(-1)
-                loss = self.torch_loss_func(torch_out.cpu(), label)
-                epoch_loss += loss.item()
-
-                self.driver.backward(loss)
-                self.driver.step()
-                self.driver.zero_grad()
-
-            current_epoch_idx += 1
-
-        # 开始测试
-        correct = 0
-        for img, label in self.test_dataset:
-
-            img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1))
-            torch_out = self.driver.test_step(img)
-            res = torch_out.softmax(-1).argmax().item()
-            label = label.item()
-            if res == label:
-                correct += 1
-
-        acc = correct / len(self.test_dataset)
-        assert acc > 0.85

tests/core/utils/_test_torch_paddle_utils.py

@@ -1,204 +0,0 @@
-import paddle
-import pytest
-import torch
-
-from fastNLP.core.utils.torch_paddle_utils import torch_paddle_move_data_to_device
-
-############################################################################
-#
-# 测试将参数中包含的所有torch和paddle张量迁移到指定设备
-#
-############################################################################
-
-@pytest.mark.torchpaddle
-class TestTorchPaddleMoveDataToDevice:
-
-    def check_gpu(self, tensor, idx):
-        """
-        检查张量是否在指定显卡上的工具函数
-        """
-
-        if isinstance(tensor, paddle.Tensor):
-            assert tensor.place.is_gpu_place()
-            assert tensor.place.gpu_device_id() == idx
-        elif isinstance(tensor, torch.Tensor):
-            assert tensor.is_cuda
-            assert tensor.device.index == idx
-
-    def check_cpu(self, tensor):
-        if isinstance(tensor, paddle.Tensor):
-            assert tensor.place.is_cpu_place()
-        elif isinstance(tensor, torch.Tensor):
-            assert not tensor.is_cuda
-
-    def test_tensor_transfer(self):
-        """
-        测试迁移单个张量
-        """
-
-        paddle_tensor = paddle.rand((3, 4, 5)).cpu()
-        res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device=None)
-        self.check_cpu(res)
-
-        res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None)
-        self.check_gpu(res, 0)
-
-        res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:1", data_device=None)
-        self.check_gpu(res, 1)
-
-        res = torch_paddle_move_data_to_device(paddle_tensor, device="cuda:0", data_device="cpu")
-        self.check_gpu(res, 0)
-
-        res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0")
-        self.check_gpu(res, 0)
-
-        res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="cuda:1")
-        self.check_gpu(res, 1)
-
-        torch_tensor = torch.rand(3, 4, 5)
-        res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device=None)
-        self.check_cpu(res)
-
-        res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device=None)
-        self.check_gpu(res, 0)
-
-        res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:1", data_device=None)
-        self.check_gpu(res, 1)
-
-        res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device="cpu")
-        self.check_gpu(res, 0)
-
-        res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:0")
-        self.check_gpu(res, 0)
-
-        res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:1")
-        self.check_gpu(res, 1)
-
-    def test_list_transfer(self):
-        """
-        测试迁移张量的列表
-        """
-
-        paddle_list = [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)]
-        res = torch_paddle_move_data_to_device(paddle_list, device=None, data_device="gpu:1")
-        assert isinstance(res, list)
-        for r in res:
-            self.check_gpu(r, 1)
-
-        res = torch_paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1")
-        assert isinstance(res, list)
-        for r in res:
-            self.check_cpu(r)
-
-        res = torch_paddle_move_data_to_device(paddle_list, device="gpu:0", data_device=None)
-        assert isinstance(res, list)
-        for r in res:
-            self.check_gpu(r, 0)
-
-        res = torch_paddle_move_data_to_device(paddle_list, device="gpu:1", data_device="cpu")
-        assert isinstance(res, list)
-        for r in res:
-            self.check_gpu(r, 1)
-
-    def test_tensor_tuple_transfer(self):
-        """
-        测试迁移张量的元组
-        """
-
-        paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] + [torch.rand((6, 4, 2)) for i in range(5)]
-        paddle_tuple = tuple(paddle_list)
-        res = torch_paddle_move_data_to_device(paddle_tuple, device=None, data_device="gpu:1")
-        assert isinstance(res, tuple)
-        for r in res:
-            self.check_gpu(r, 1)
-
-        res = torch_paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1")
-        assert isinstance(res, tuple)
-        for r in res:
-            self.check_cpu(r)
-
-        res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:0", data_device=None)
-        assert isinstance(res, tuple)
-        for r in res:
-            self.check_gpu(r, 0)
-
-        res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:1", data_device="cpu")
-        assert isinstance(res, tuple)
-        for r in res:
-            self.check_gpu(r, 1)
-
-    def test_dict_transfer(self):
-        """
-        测试迁移复杂的字典结构
-        """
-
-        paddle_dict = {
-            "torch_tensor": torch.rand((3, 4)),
-            "torch_list": [torch.rand((6, 4, 2)) for i in range(10)],
-            "dict":{
-                "list": [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)],
-                "torch_tensor": torch.rand((3, 4)),
-                "paddle_tensor": paddle.rand((3, 4))
-            },
-            "paddle_tensor": paddle.rand((3, 4)),
-            "list": [paddle.rand((6, 4, 2)) for i in range(10)] ,
-            "int": 2,
-            "string": "test string"
-        }
-
-        res = torch_paddle_move_data_to_device(paddle_dict, device="gpu:0", data_device=None)
-        assert isinstance(res, dict)
-        self.check_gpu(res["torch_tensor"], 0)
-        self.check_gpu(res["paddle_tensor"], 0)
-        assert isinstance(res["torch_list"], list)
-        for t in res["torch_list"]:
-            self.check_gpu(t, 0)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_gpu(t, 0)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_gpu(t, 0)
-        self.check_gpu(res["dict"]["torch_tensor"], 0)
-        self.check_gpu(res["dict"]["paddle_tensor"], 0)
-
-        res = torch_paddle_move_data_to_device(paddle_dict, device=None, data_device="gpu:1")
-        assert isinstance(res, dict)
-        self.check_gpu(res["torch_tensor"], 1)
-        self.check_gpu(res["paddle_tensor"], 1)
-        assert isinstance(res["torch_list"], list)
-        for t in res["torch_list"]:
-            self.check_gpu(t, 1)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_gpu(t, 1)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_gpu(t, 1)
-        self.check_gpu(res["dict"]["torch_tensor"], 1)
-        self.check_gpu(res["dict"]["paddle_tensor"], 1)
-
-        res = torch_paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0")
-        assert isinstance(res, dict)
-        self.check_cpu(res["torch_tensor"])
-        self.check_cpu(res["paddle_tensor"])
-        assert isinstance(res["torch_list"], list)
-        for t in res["torch_list"]:
-            self.check_cpu(t)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_cpu(t)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_cpu(t)
-        self.check_cpu(res["dict"]["torch_tensor"])
-        self.check_cpu(res["dict"]["paddle_tensor"])

tests/modules/mix_modules/_test_mix_module.py

@@ -1,378 +0,0 @@
-import pytest
-import os
-from itertools import chain
-
-import torch
-import paddle
-from paddle.io import Dataset, DataLoader
-import numpy as np
-
-from fastNLP.modules.mix_modules.mix_module import MixModule
-from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle
-from fastNLP.envs.distributed import rank_zero_rm
-
-
-############################################################################
-#
-# 测试类的基本功能
-#
-############################################################################
-
-class MixModuleForTest(MixModule):
-    def __init__(self):
-        super(MixModuleForTest, self).__init__()
-
-        self.torch_fc1 = torch.nn.Linear(10, 10)
-        self.torch_softmax = torch.nn.Softmax(0)
-        self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3)
-        self.torch_tensor = torch.ones(3, 3)
-        self.torch_param = torch.nn.Parameter(torch.ones(4, 4))
-
-        self.paddle_fc1 = paddle.nn.Linear(10, 10)
-        self.paddle_softmax = paddle.nn.Softmax(0)
-        self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3)
-        self.paddle_tensor = paddle.ones((4, 4))
-
-class TorchModuleForTest(torch.nn.Module):
-    def __init__(self):
-        super(TorchModuleForTest, self).__init__()
-
-        self.torch_fc1 = torch.nn.Linear(10, 10)
-        self.torch_softmax = torch.nn.Softmax(0)
-        self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3)
-        self.torch_tensor = torch.ones(3, 3)
-        self.torch_param = torch.nn.Parameter(torch.ones(4, 4))
-
-class PaddleModuleForTest(paddle.nn.Layer):
-    def __init__(self):
-        super(PaddleModuleForTest, self).__init__()
-
-        self.paddle_fc1 = paddle.nn.Linear(10, 10)
-        self.paddle_softmax = paddle.nn.Softmax(0)
-        self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3)
-        self.paddle_tensor = paddle.ones((4, 4))
-
-
-@pytest.mark.torchpaddle
-class TestTorchPaddleMixModule:
-
-    def setup_method(self):
-
-        self.model = MixModuleForTest()
-        self.torch_model = TorchModuleForTest()
-        self.paddle_model = PaddleModuleForTest()
-
-    def test_to(self):
-        """
-        测试混合模型的to函数
-        """
-        
-        self.model.to("cuda")
-        self.torch_model.to("cuda")
-        self.paddle_model.to("gpu")
-        self.if_device_correct("cuda")
-
-        self.model.to("cuda:2")
-        self.torch_model.to("cuda:2")
-        self.paddle_model.to("gpu:2")
-        self.if_device_correct("cuda:2")
-
-        self.model.to("gpu:1")
-        self.torch_model.to("cuda:1")
-        self.paddle_model.to("gpu:1")
-        self.if_device_correct("cuda:1")
-
-        self.model.to("cpu")
-        self.torch_model.to("cpu")
-        self.paddle_model.to("cpu")
-        self.if_device_correct("cpu")
-
-    def test_train_eval(self):
-        """
-        测试train和eval函数
-        """
-        
-        self.model.eval()
-        self.if_training_correct(False)
-
-        self.model.train()
-        self.if_training_correct(True)
-
-    def test_parameters(self):
-        """
-        测试parameters()函数，由于初始化是随机的，目前仅比较得到结果的长度
-        """
-        mix_params = []
-        params = []
-
-        for value in self.model.named_parameters():
-            mix_params.append(value)
-
-        for value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()):
-            params.append(value)
-
-        assert len(params) == len(mix_params)
-
-    def test_named_parameters(self):
-        """
-        测试named_parameters函数
-        """
-        
-        mix_param_names = []
-        param_names = []
-
-        for name, value in self.model.named_parameters():
-            mix_param_names.append(name)
-
-        for name, value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()):
-            param_names.append(name)
-
-        assert sorted(param_names) == sorted(mix_param_names)
-
-    def test_torch_named_parameters(self):
-        """
-        测试对torch参数的提取
-        """
-        
-        mix_param_names = []
-        param_names = []
-
-        for name, value in self.model.named_parameters(backend="torch"):
-            mix_param_names.append(name)
-
-        for name, value in self.torch_model.named_parameters():
-            param_names.append(name)
-
-        assert sorted(param_names) == sorted(mix_param_names)
-
-    def test_paddle_named_parameters(self):
-        """
-        测试对paddle参数的提取
-        """
-        
-        mix_param_names = []
-        param_names = []
-
-        for name, value in self.model.named_parameters(backend="paddle"):
-            mix_param_names.append(name)
-
-        for name, value in self.paddle_model.named_parameters():
-            param_names.append(name)
-
-        assert sorted(param_names) == sorted(mix_param_names)
-
-    def test_torch_state_dict(self):
-        """
-        测试提取torch的state dict
-        """
-        torch_dict = self.torch_model.state_dict()
-        mix_dict = self.model.state_dict(backend="torch")
-
-        assert sorted(torch_dict.keys()) == sorted(mix_dict.keys())
-
-    def test_paddle_state_dict(self):
-        """
-        测试提取paddle的state dict
-        """
-        paddle_dict = self.paddle_model.state_dict()
-        mix_dict = self.model.state_dict(backend="paddle")
-
-        # TODO 测试程序会显示passed后显示paddle的异常退出信息
-        assert sorted(paddle_dict.keys()) == sorted(mix_dict.keys())
-
-    def test_state_dict(self):
-        """
-        测试提取所有的state dict
-        """
-        all_dict = self.torch_model.state_dict()
-        all_dict.update(self.paddle_model.state_dict())
-        mix_dict = self.model.state_dict()
-
-        # TODO 测试程序会显示passed后显示paddle的异常退出信息
-        assert sorted(all_dict.keys()) == sorted(mix_dict.keys())
-
-    def test_load_state_dict(self):
-        """
-        测试load_state_dict函数
-        """
-        state_dict = self.model.state_dict()
-
-        new_model = MixModuleForTest()
-        new_model.load_state_dict(state_dict)
-        new_state_dict = new_model.state_dict()
-
-        for name, value in state_dict.items():
-            state_dict[name] = value.tolist()
-        for name, value in new_state_dict.items():
-            new_state_dict[name] = value.tolist()
-
-        # self.assertDictEqual(state_dict, new_state_dict)
-
-    def test_save_and_load_state_dict(self):
-        """
-        测试save_state_dict_to_file和load_state_dict_from_file函数
-        """
-        path = "model"
-        try:
-            self.model.save_state_dict_to_file(path)
-            new_model = MixModuleForTest()
-            new_model.load_state_dict_from_file(path)
-
-            state_dict = self.model.state_dict()
-            new_state_dict = new_model.state_dict()
-
-            for name, value in state_dict.items():
-                state_dict[name] = value.tolist()
-            for name, value in new_state_dict.items():
-                new_state_dict[name] = value.tolist()
-
-            # self.assertDictEqual(state_dict, new_state_dict)
-        finally:
-            rank_zero_rm(path)
-
-    def if_device_correct(self, device):
-
-
-        assert self.model.torch_fc1.weight.device == self.torch_model.torch_fc1.weight.device
-        assert self.model.torch_conv2d1.weight.device == self.torch_model.torch_fc1.bias.device
-        assert self.model.torch_conv2d1.bias.device == self.torch_model.torch_conv2d1.bias.device
-        assert self.model.torch_tensor.device == self.torch_model.torch_tensor.device
-        assert self.model.torch_param.device == self.torch_model.torch_param.device
-
-        if device == "cpu":
-            assert self.model.paddle_fc1.weight.place.is_cpu_place()
-            assert self.model.paddle_fc1.bias.place.is_cpu_place()
-            assert self.model.paddle_conv2d1.weight.place.is_cpu_place()
-            assert self.model.paddle_conv2d1.bias.place.is_cpu_place()
-            assert self.model.paddle_tensor.place.is_cpu_place()
-        elif device.startswith("cuda"):
-            assert self.model.paddle_fc1.weight.place.is_gpu_place()
-            assert self.model.paddle_fc1.bias.place.is_gpu_place()
-            assert self.model.paddle_conv2d1.weight.place.is_gpu_place()
-            assert self.model.paddle_conv2d1.bias.place.is_gpu_place()
-            assert self.model.paddle_tensor.place.is_gpu_place()
-
-            assert self.model.paddle_fc1.weight.place.gpu_device_id() == self.paddle_model.paddle_fc1.weight.place.gpu_device_id()
-            assert self.model.paddle_fc1.bias.place.gpu_device_id() == self.paddle_model.paddle_fc1.bias.place.gpu_device_id()
-            assert self.model.paddle_conv2d1.weight.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.weight.place.gpu_device_id()
-            assert self.model.paddle_conv2d1.bias.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.bias.place.gpu_device_id()
-            assert self.model.paddle_tensor.place.gpu_device_id() == self.paddle_model.paddle_tensor.place.gpu_device_id()
-        else:
-            raise NotImplementedError
-
-    def if_training_correct(self, training):
-
-        assert self.model.torch_fc1.training == training
-        assert self.model.torch_softmax.training == training
-        assert self.model.torch_conv2d1.training == training
-
-        assert self.model.paddle_fc1.training == training
-        assert self.model.paddle_softmax.training == training
-        assert self.model.paddle_conv2d1.training == training
-
-
-############################################################################
-#
-# 测试在MNIST数据集上的表现
-#
-############################################################################
-
-class MNISTDataset(Dataset):
-    def __init__(self, dataset):
-
-        self.dataset = [
-            (
-                np.array(img).astype('float32').reshape(-1),
-                label
-            ) for img, label in dataset
-        ]
-
-    def __getitem__(self, idx):
-        return self.dataset[idx]
-
-    def __len__(self):
-        return len(self.dataset)
-
-class MixMNISTModel(MixModule):
-    def __init__(self):
-        super(MixMNISTModel, self).__init__()
-
-        self.fc1 = paddle.nn.Linear(784, 64)
-        self.fc2 = paddle.nn.Linear(64, 32)
-        self.fc3 = torch.nn.Linear(32, 10)
-        self.fc4 = torch.nn.Linear(10, 10)
-
-    def forward(self, x):
-
-        paddle_out = self.fc1(x)
-        paddle_out = self.fc2(paddle_out)
-        torch_in = paddle2torch(paddle_out)
-        torch_out = self.fc3(torch_in)
-        torch_out = self.fc4(torch_out)
-
-        return torch_out
-
-@pytest.mark.torchpaddle
-class TestMNIST:
-
-    @classmethod
-    def setup_class(self):
-
-        self.train_dataset = paddle.vision.datasets.MNIST(mode='train')
-        self.test_dataset = paddle.vision.datasets.MNIST(mode='test')
-        self.train_dataset = MNISTDataset(self.train_dataset)
-
-        self.lr = 0.0003
-        self.epochs = 20
-
-        self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True)
-
-    def setup_method(self):
-        
-        self.model = MixMNISTModel().to("cuda")
-        self.torch_loss_func = torch.nn.CrossEntropyLoss()
-
-        self.torch_opt = torch.optim.Adam(self.model.parameters(backend="torch"), self.lr)
-        self.paddle_opt = paddle.optimizer.Adam(parameters=self.model.parameters(backend="paddle"), learning_rate=self.lr)
-
-    def test_case1(self):
-
-        # 开始训练
-        for epoch in range(self.epochs):
-            epoch_loss, batch = 0, 0
-            for batch, (img, label) in enumerate(self.dataloader):
-
-                img = paddle.to_tensor(img).cuda()
-                torch_out = self.model(img)
-                label = torch.from_numpy(label.numpy()).reshape(-1)
-                loss = self.torch_loss_func(torch_out.cpu(), label)
-                epoch_loss += loss.item()
-
-                loss.backward()
-                self.torch_opt.step()
-                self.paddle_opt.step()
-                self.torch_opt.zero_grad()
-                self.paddle_opt.clear_grad()
-
-        else:
-            assert epoch_loss / (batch + 1) < 0.3
-
-        # 开始测试
-        correct = 0
-        for img, label in self.test_dataset:
-
-            img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1))
-            torch_out = self.model(img)
-            res = torch_out.softmax(-1).argmax().item()
-            label = label.item()
-            if res == label:
-                correct += 1
-
-        acc = correct / len(self.test_dataset)
-        assert acc > 0.85
-
-############################################################################
-#
-# 测试在ERNIE中文数据集上的表现
-#
-############################################################################

tests/modules/mix_modules/_test_utils.py

@@ -1,435 +0,0 @@
-import unittest
-import os
-
-os.environ["log_silent"] = "1"
-import torch
-import paddle
-import jittor
-
-from fastNLP.modules.mix_modules.utils import (
-    paddle2torch,
-    torch2paddle,
-    jittor2torch,
-    torch2jittor,
-)
-
-############################################################################
-#
-# 测试paddle到torch的转换
-#
-############################################################################
-
-class Paddle2TorchTestCase(unittest.TestCase):
-
-    def check_torch_tensor(self, tensor, device, requires_grad):
-        """
-        检查张量设备和梯度情况的工具函数
-        """
-
-        assert isinstance(tensor, torch.Tensor)
-        assert tensor.device == torch.device(device)
-        assert tensor.requires_grad == requires_grad
-
-    def test_gradient(self):
-        """
-        测试张量转换后的反向传播是否正确
-        """
-
-        x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0], stop_gradient=False)
-        y = paddle2torch(x)
-        z = 3 * (y ** 2)
-        z.sum().backward()
-        assert y.grad.tolist() == [6, 12, 18, 24, 30]
-
-    def test_tensor_transfer(self):
-        """
-        测试单个张量的设备和梯度转换是否正确
-        """
-
-        paddle_tensor = paddle.rand((3, 4, 5)).cpu()
-        res = paddle2torch(paddle_tensor)
-        self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient)
-
-        res = paddle2torch(paddle_tensor, target_device="cuda:2", no_gradient=None)
-        self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient)
-
-        res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=True)
-        self.check_torch_tensor(res, "cuda:1", False)
-
-        res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=False)
-        self.check_torch_tensor(res, "cuda:1", True)
-
-    def test_list_transfer(self):
-        """
-        测试张量列表的转换
-        """
-
-        paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)]
-        res = paddle2torch(paddle_list)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_torch_tensor(t, "cuda:1", False)
-
-        res = paddle2torch(paddle_list, target_device="cpu", no_gradient=False)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_torch_tensor(t, "cpu", True)
-
-    def test_tensor_tuple_transfer(self):
-        """
-        测试张量元组的转换
-        """
-
-        paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)]
-        paddle_tuple = tuple(paddle_list)
-        res = paddle2torch(paddle_tuple)
-        assert isinstance(res, tuple)
-        for t in res:
-            self.check_torch_tensor(t, "cuda:1", False)
-
-    def test_dict_transfer(self):
-        """
-        测试包含复杂结构的字典的转换
-        """
-
-        paddle_dict = {
-            "tensor": paddle.rand((3, 4)).cuda(0),
-            "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)],
-            "dict":{
-                "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)],
-                "tensor": paddle.rand((3, 4)).cuda(0)
-            },
-            "int": 2,
-            "string": "test string"
-        }
-        res = paddle2torch(paddle_dict)
-        assert isinstance(res, dict)
-        self.check_torch_tensor(res["tensor"], "cuda:0", False)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_torch_tensor(t, "cuda:0", False)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_torch_tensor(t, "cuda:0", False)
-        self.check_torch_tensor(res["dict"]["tensor"], "cuda:0", False)
-
-
-############################################################################
-#
-# 测试torch到paddle的转换
-#
-############################################################################
-
-class Torch2PaddleTestCase(unittest.TestCase):
-
-    def check_paddle_tensor(self, tensor, device, stop_gradient):
-        """
-        检查得到的paddle张量设备和梯度情况的工具函数
-        """
-
-        assert isinstance(tensor, paddle.Tensor)
-        if device == "cpu":
-            assert tensor.place.is_cpu_place()
-        elif device.startswith("gpu"):
-            paddle_device = paddle.device._convert_to_place(device)
-            assert tensor.place.is_gpu_place()
-            if hasattr(tensor.place, "gpu_device_id"):
-                # paddle中，有两种Place
-                # paddle.fluid.core.Place是创建Tensor时使用的类型
-                # 有函数gpu_device_id获取设备
-                assert tensor.place.gpu_device_id() == paddle_device.get_device_id()
-            else:
-                # 通过_convert_to_place得到的是paddle.CUDAPlace
-                # 通过get_device_id获取设备
-                assert tensor.place.get_device_id() == paddle_device.get_device_id()
-        else:
-            raise NotImplementedError
-        assert tensor.stop_gradient == stop_gradient
-
-    def test_gradient(self):
-        """
-        测试转换后梯度的反向传播
-        """
-
-        x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True)
-        y = torch2paddle(x)
-        z = 3 * (y ** 2)
-        z.sum().backward()
-        assert y.grad.tolist() == [6, 12, 18, 24, 30]
-
-    def test_tensor_transfer(self):
-        """
-        测试单个张量的转换
-        """
-
-        torch_tensor = torch.rand((3, 4, 5))
-        res = torch2paddle(torch_tensor)
-        self.check_paddle_tensor(res, "cpu", True)
-
-        res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=None)
-        self.check_paddle_tensor(res, "gpu:2", True)
-
-        res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=True)
-        self.check_paddle_tensor(res, "gpu:2", True)
-
-        res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=False)
-        self.check_paddle_tensor(res, "gpu:2", False)
-
-    def test_tensor_list_transfer(self):
-        """
-        测试张量列表的转换
-        """
-
-        torch_list = [torch.rand(6, 4, 2) for i in range(10)]
-        res = torch2paddle(torch_list)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_paddle_tensor(t, "cpu", True)
-
-        res = torch2paddle(torch_list, target_device="gpu:1", no_gradient=False)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_paddle_tensor(t, "gpu:1", False)
-
-    def test_tensor_tuple_transfer(self):
-        """
-        测试张量元组的转换
-        """
-        
-        torch_list = [torch.rand(6, 4, 2) for i in range(10)]
-        torch_tuple = tuple(torch_list)
-        res = torch2paddle(torch_tuple, target_device="cpu")
-        assert isinstance(res, tuple)
-        for t in res:
-            self.check_paddle_tensor(t, "cpu", True)
-
-    def test_dict_transfer(self):
-        """
-        测试复杂的字典结构的转换
-        """
-
-        torch_dict = {
-            "tensor": torch.rand((3, 4)),
-            "list": [torch.rand(6, 4, 2) for i in range(10)],
-            "dict":{
-                "list": [torch.rand(6, 4, 2) for i in range(10)],
-                "tensor": torch.rand((3, 4))
-            },
-            "int": 2,
-            "string": "test string"
-        }
-        res = torch2paddle(torch_dict)
-        assert isinstance(res, dict)
-        self.check_paddle_tensor(res["tensor"], "cpu", True)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_paddle_tensor(t, "cpu", True)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_paddle_tensor(t, "cpu", True)
-        self.check_paddle_tensor(res["dict"]["tensor"], "cpu", True)
-
-
-############################################################################
-#
-# 测试jittor到torch的转换
-#
-############################################################################
-
-class Jittor2TorchTestCase(unittest.TestCase):
-
-    def check_torch_tensor(self, tensor, device, requires_grad):
-        """
-        检查得到的torch张量的工具函数
-        """
-
-        assert isinstance(tensor, torch.Tensor)
-        if device == "cpu":
-            assert not tensor.is_cuda
-        else:
-            assert tensor.device == torch.device(device)
-        assert tensor.requires_grad == requires_grad
-
-    def test_var_transfer(self):
-        """
-        测试单个Jittor Var的转换
-        """
-
-        jittor_var = jittor.rand((3, 4, 5))
-        res = jittor2torch(jittor_var)
-        self.check_torch_tensor(res, "cpu", True)
-
-        res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=None)
-        self.check_torch_tensor(res, "cuda:2", True)
-
-        res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=True)
-        self.check_torch_tensor(res, "cuda:2", False)
-
-        res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=False)
-        self.check_torch_tensor(res, "cuda:2", True)
-
-    def test_var_list_transfer(self):
-        """
-        测试Jittor列表的转换
-        """
-
-        jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)]
-        res = jittor2torch(jittor_list)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_torch_tensor(t, "cpu", True)
-
-        res = jittor2torch(jittor_list, target_device="cuda:1", no_gradient=False)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_torch_tensor(t, "cuda:1", True)
-
-    def test_var_tuple_transfer(self):
-        """
-        测试Jittor变量元组的转换
-        """
-
-        jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)]
-        jittor_tuple = tuple(jittor_list)
-        res = jittor2torch(jittor_tuple, target_device="cpu")
-        assert isinstance(res, tuple)
-        for t in res:
-            self.check_torch_tensor(t, "cpu", True)
-
-    def test_dict_transfer(self):
-        """
-        测试字典结构的转换
-        """
-
-        jittor_dict = {
-            "tensor": jittor.rand((3, 4)),
-            "list": [jittor.rand(6, 4, 2) for i in range(10)],
-            "dict":{
-                "list": [jittor.rand(6, 4, 2) for i in range(10)],
-                "tensor": jittor.rand((3, 4))
-            },
-            "int": 2,
-            "string": "test string"
-        }
-        res = jittor2torch(jittor_dict)
-        assert isinstance(res, dict)
-        self.check_torch_tensor(res["tensor"], "cpu", True)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_torch_tensor(t, "cpu", True)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_torch_tensor(t, "cpu", True)
-        self.check_torch_tensor(res["dict"]["tensor"], "cpu", True)
-
-
-############################################################################
-#
-# 测试torch到jittor的转换
-#
-############################################################################
-
-class Torch2JittorTestCase(unittest.TestCase):
-
-    def check_jittor_var(self, var, requires_grad):
-        """
-        检查得到的Jittor Var梯度情况的工具函数
-        """
-
-        assert isinstance(var, jittor.Var)
-        assert var.requires_grad == requires_grad
-
-    def test_gradient(self):
-        """
-        测试反向传播的梯度
-        """
-
-        x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True)
-        y = torch2jittor(x)
-        z = 3 * (y ** 2)
-        grad = jittor.grad(z, y)
-        assert grad.tolist() == [6.0, 12.0, 18.0, 24.0, 30.0]
-
-    def test_tensor_transfer(self):
-        """
-        测试单个张量转换为Jittor
-        """
-
-        torch_tensor = torch.rand((3, 4, 5))
-        res = torch2jittor(torch_tensor)
-        self.check_jittor_var(res, False)
-
-        res = torch2jittor(torch_tensor, no_gradient=None)
-        self.check_jittor_var(res, False)
-
-        res = torch2jittor(torch_tensor, no_gradient=True)
-        self.check_jittor_var(res, False)
-
-        res = torch2jittor(torch_tensor, no_gradient=False)
-        self.check_jittor_var(res, True)
-
-    def test_tensor_list_transfer(self):
-        """
-        测试张量列表的转换
-        """
-
-        torch_list = [torch.rand((6, 4, 2)) for i in range(10)]
-        res = torch2jittor(torch_list)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_jittor_var(t, False)
-
-        res = torch2jittor(torch_list, no_gradient=False)
-        assert isinstance(res, list)
-        for t in res:
-            self.check_jittor_var(t, True)
-
-    def test_tensor_tuple_transfer(self):
-        """
-        测试张量元组的转换
-        """
-
-        torch_list = [torch.rand((6, 4, 2)) for i in range(10)]
-        torch_tuple = tuple(torch_list)
-        res = torch2jittor(torch_tuple)
-        assert isinstance(res, tuple)
-        for t in res:
-            self.check_jittor_var(t, False)
-
-    def test_dict_transfer(self):
-        """
-        测试字典结构的转换
-        """
-
-        torch_dict = {
-            "tensor": torch.rand((3, 4)),
-            "list": [torch.rand(6, 4, 2) for i in range(10)],
-            "dict":{
-                "list": [torch.rand(6, 4, 2) for i in range(10)],
-                "tensor": torch.rand((3, 4))
-            },
-            "int": 2,
-            "string": "test string"
-        }
-        res = torch2jittor(torch_dict)
-        assert isinstance(res, dict)
-        self.check_jittor_var(res["tensor"], False)
-        assert isinstance(res["list"], list)
-        for t in res["list"]:
-            self.check_jittor_var(t, False)
-        assert isinstance(res["int"], int)
-        assert isinstance(res["string"], str)
-        assert isinstance(res["dict"], dict)
-        assert isinstance(res["dict"]["list"], list)
-        for t in res["dict"]["list"]:
-            self.check_jittor_var(t, False)
-        self.check_jittor_var(res["dict"]["tensor"], False)