Browse Source

[to #42322933] change dummy dataset to msdataset

1. change dummy dataset to msdataset
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9586561

    * change dummpy dataset to msdataset

* add pre-commit ignore

* Merge commit 'e93339ea877b93fa0c1b9ebfeee8877f78facb0e' into feat/ms_dataset_case

* Merge commit '34840fc5d8a8ee8cd1278efea913d42db522f9c8' into feat/ms_dataset_case

* remove useless ip hosts.

* Merge commit '47dda0a5f9b4b4466177d9acae097a53f8bea8f7' into feat/ms_dataset_case

* Merge commit '21de1e7db035843c6c2caccf382a6e4f7071f96b' into feat/ms_dataset_case
master
yuze.zyz 3 years ago
parent
commit
7d348b9ae8
1 changed files with 2 additions and 14 deletions
  1. +2
    -14
      tests/trainers/test_trainer_with_nlp.py

+ 2
- 14
tests/trainers/test_trainer_with_nlp.py View File

@@ -24,20 +24,8 @@ class TestTrainerWithNlp(unittest.TestCase):
os.makedirs(self.tmp_dir)

# todo: Replace below scripts with MsDataset.load when the formal dataset service is ready
from datasets import Dataset
dataset_dict = {
'sentence1': [
'This is test sentence1-1', 'This is test sentence2-1',
'This is test sentence3-1'
],
'sentence2': [
'This is test sentence1-2', 'This is test sentence2-2',
'This is test sentence3-2'
],
'label': [0, 1, 1]
}
dataset = Dataset.from_dict(dataset_dict)
self.dataset = MsDataset.from_hf_dataset(dataset)
self.dataset = MsDataset.load(
'afqmc_small', namespace='userxiaoming', split='train')

def tearDown(self):
shutil.rmtree(self.tmp_dir)


Loading…
Cancel
Save