| @@ -1,10 +1,12 @@ | |||||
| import pytest | import pytest | ||||
| import numpy as np | |||||
| from fastNLP.core.dataloaders.paddle_dataloader.fdl import PaddleDataLoader | from fastNLP.core.dataloaders.paddle_dataloader.fdl import PaddleDataLoader | ||||
| from fastNLP.core.dataset import DataSet | from fastNLP.core.dataset import DataSet | ||||
| from paddle.io import Dataset, DataLoader | |||||
| import numpy as np | |||||
| import paddle | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| from paddle.io import Dataset, DataLoader | |||||
| import paddle | |||||
| class RandomDataset(Dataset): | class RandomDataset(Dataset): | ||||
| @@ -14,9 +14,10 @@ from fastNLP.core.drivers.paddle_driver.dist_utils import ( | |||||
| ) | ) | ||||
| from fastNLP.core.drivers.paddle_driver.fleet_launcher import FleetLauncher | from fastNLP.core.drivers.paddle_driver.fleet_launcher import FleetLauncher | ||||
| from tests.helpers.utils import magic_argv_env_context | from tests.helpers.utils import magic_argv_env_context | ||||
| import paddle | |||||
| import paddle.distributed as dist | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| import paddle.distributed as dist | |||||
| @pytest.mark.paddle | @pytest.mark.paddle | ||||
| class TestDistUtilsTools: | class TestDistUtilsTools: | ||||
| @@ -13,10 +13,11 @@ from tests.helpers.models.paddle_model import PaddleNormalModel_Classification_1 | |||||
| from tests.helpers.datasets.paddle_data import PaddleNormalDataset, PaddleRandomMaxDataset | from tests.helpers.datasets.paddle_data import PaddleNormalDataset, PaddleRandomMaxDataset | ||||
| from tests.helpers.utils import magic_argv_env_context | from tests.helpers.utils import magic_argv_env_context | ||||
| from fastNLP.core import rank_zero_rm | from fastNLP.core import rank_zero_rm | ||||
| import paddle | |||||
| import paddle.distributed as dist | |||||
| from paddle.io import DataLoader, BatchSampler | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| import paddle.distributed as dist | |||||
| from paddle.io import DataLoader, BatchSampler | |||||
| def generate_driver(num_labels, feature_dimension, device=[0,1], fp16=False, output_from_new_proc="only_error"): | def generate_driver(num_labels, feature_dimension, device=[0,1], fp16=False, output_from_new_proc="only_error"): | ||||
| paddle_model = PaddleNormalModel_Classification_1(num_labels, feature_dimension) | paddle_model = PaddleNormalModel_Classification_1(num_labels, feature_dimension) | ||||
| @@ -5,8 +5,9 @@ from fastNLP.core.drivers.paddle_driver.initialize_paddle_driver import initiali | |||||
| from fastNLP.envs import get_gpu_count | from fastNLP.envs import get_gpu_count | ||||
| from tests.helpers.models.paddle_model import PaddleNormalModel_Classification_1 | from tests.helpers.models.paddle_model import PaddleNormalModel_Classification_1 | ||||
| from tests.helpers.utils import magic_argv_env_context | from tests.helpers.utils import magic_argv_env_context | ||||
| import paddle | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| @pytest.mark.paddle | @pytest.mark.paddle | ||||
| def test_incorrect_driver(): | def test_incorrect_driver(): | ||||
| @@ -8,10 +8,12 @@ from tests.helpers.datasets.paddle_data import PaddleNormalDataset, PaddleRandom | |||||
| from tests.helpers.datasets.torch_data import TorchNormalDataset | from tests.helpers.datasets.torch_data import TorchNormalDataset | ||||
| from tests.helpers.models.torch_model import TorchNormalModel_Classification_1 | from tests.helpers.models.torch_model import TorchNormalModel_Classification_1 | ||||
| from fastNLP.core import rank_zero_rm | from fastNLP.core import rank_zero_rm | ||||
| import paddle | |||||
| from paddle.io import DataLoader, BatchSampler | |||||
| import torch | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| from paddle.io import DataLoader, BatchSampler | |||||
| if _NEED_IMPORT_TORCH: | |||||
| import torch | |||||
| ############################################################################ | ############################################################################ | ||||
| # | # | ||||
| @@ -7,9 +7,10 @@ from fastNLP.core.drivers.paddle_driver.utils import ( | |||||
| replace_sampler, | replace_sampler, | ||||
| ) | ) | ||||
| from fastNLP.core.samplers import RandomBatchSampler, RandomSampler | from fastNLP.core.samplers import RandomBatchSampler, RandomSampler | ||||
| import paddle | |||||
| from paddle.io import DataLoader, BatchSampler | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| from paddle.io import DataLoader, BatchSampler | |||||
| from tests.helpers.datasets.paddle_data import PaddleNormalDataset | from tests.helpers.datasets.paddle_data import PaddleNormalDataset | ||||
| @@ -8,10 +8,12 @@ from tests.helpers.datasets.torch_data import TorchNormalDataset, TorchArgMaxDat | |||||
| from tests.helpers.datasets.paddle_data import PaddleNormalDataset | from tests.helpers.datasets.paddle_data import PaddleNormalDataset | ||||
| from tests.helpers.models.paddle_model import PaddleNormalModel_Classification_1 | from tests.helpers.models.paddle_model import PaddleNormalModel_Classification_1 | ||||
| from fastNLP.core import rank_zero_rm | from fastNLP.core import rank_zero_rm | ||||
| import torch | |||||
| from torch.utils.data import DataLoader, BatchSampler | |||||
| import paddle | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH | |||||
| if _NEED_IMPORT_TORCH: | |||||
| import torch | |||||
| from torch.utils.data import DataLoader, BatchSampler | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| def dataloader_with_randombatchsampler(dataset, batch_size, shuffle, drop_last): | def dataloader_with_randombatchsampler(dataset, batch_size, shuffle, drop_last): | ||||
| """ | """ | ||||
| @@ -1,12 +1,14 @@ | |||||
| import os | import os | ||||
| import pytest | import pytest | ||||
| import paddle | |||||
| import paddle.distributed | |||||
| import paddle.distributed.fleet.base.role_maker as role_maker | |||||
| import paddle.distributed.fleet as fleet | |||||
| from fastNLP.core.metrics import Accuracy | from fastNLP.core.metrics import Accuracy | ||||
| from fastNLP.core.drivers.paddle_driver.fleet_launcher import FleetLauncher | from fastNLP.core.drivers.paddle_driver.fleet_launcher import FleetLauncher | ||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| import paddle.distributed | |||||
| import paddle.distributed.fleet.base.role_maker as role_maker | |||||
| import paddle.distributed.fleet as fleet | |||||
| ############################################################################ | ############################################################################ | ||||
| @@ -1,7 +1,9 @@ | |||||
| import pytest | import pytest | ||||
| import paddle | |||||
| from fastNLP.core.utils.paddle_utils import paddle_to, paddle_move_data_to_device | from fastNLP.core.utils.paddle_utils import paddle_to, paddle_move_data_to_device | ||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| ############################################################################ | ############################################################################ | ||||
| @@ -1,7 +1,10 @@ | |||||
| import paddle | |||||
| from paddle.io import Dataset | |||||
| import numpy as np | import numpy as np | ||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| from paddle.io import Dataset | |||||
| class PaddleNormalDataset(Dataset): | class PaddleNormalDataset(Dataset): | ||||
| def __init__(self, num_of_data=1000): | def __init__(self, num_of_data=1000): | ||||
| @@ -1,5 +1,7 @@ | |||||
| import paddle | |||||
| import paddle.nn as nn | |||||
| from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
| if _NEED_IMPORT_PADDLE: | |||||
| import paddle | |||||
| import paddle.nn as nn | |||||
| class PaddleNormalModel_Classification_1(paddle.nn.Layer): | class PaddleNormalModel_Classification_1(paddle.nn.Layer): | ||||
| """ | """ | ||||