From 7d348b9ae87084cced9557e56ef9d1f05ecfa371 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 2 Aug 2022 21:07:47 +0800 Subject: [PATCH] [to #42322933] change dummy dataset to msdataset 1. change dummy dataset to msdataset Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9586561 * change dummpy dataset to msdataset * add pre-commit ignore * Merge commit 'e93339ea877b93fa0c1b9ebfeee8877f78facb0e' into feat/ms_dataset_case * Merge commit '34840fc5d8a8ee8cd1278efea913d42db522f9c8' into feat/ms_dataset_case * remove useless ip hosts. * Merge commit '47dda0a5f9b4b4466177d9acae097a53f8bea8f7' into feat/ms_dataset_case * Merge commit '21de1e7db035843c6c2caccf382a6e4f7071f96b' into feat/ms_dataset_case --- tests/trainers/test_trainer_with_nlp.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/trainers/test_trainer_with_nlp.py b/tests/trainers/test_trainer_with_nlp.py index a28bc9e9..603d6e5b 100644 --- a/tests/trainers/test_trainer_with_nlp.py +++ b/tests/trainers/test_trainer_with_nlp.py @@ -24,20 +24,8 @@ class TestTrainerWithNlp(unittest.TestCase): os.makedirs(self.tmp_dir) # todo: Replace below scripts with MsDataset.load when the formal dataset service is ready - from datasets import Dataset - dataset_dict = { - 'sentence1': [ - 'This is test sentence1-1', 'This is test sentence2-1', - 'This is test sentence3-1' - ], - 'sentence2': [ - 'This is test sentence1-2', 'This is test sentence2-2', - 'This is test sentence3-2' - ], - 'label': [0, 1, 1] - } - dataset = Dataset.from_dict(dataset_dict) - self.dataset = MsDataset.from_hf_dataset(dataset) + self.dataset = MsDataset.load( + 'afqmc_small', namespace='userxiaoming', split='train') def tearDown(self): shutil.rmtree(self.tmp_dir)