From 064f1041a982a6745cf182385d20751a50757a5f Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 3 Aug 2022 20:31:10 +0800 Subject: [PATCH] [to #42322933] Refine the nlp examples of finetuning 1. Refine the nlp finetune examples 2. Remove some useless code Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9631158 --- modelscope/msdatasets/ms_dataset.py | 7 ------- modelscope/trainers/hooks/evaluation_hook.py | 1 - tests/trainers/test_trainer_with_nlp.py | 1 - 3 files changed, 9 deletions(-) diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index f6896e4a..8174d054 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -517,10 +517,3 @@ class MsDataset: def to_hf_dataset(self) -> Dataset: self._hf_ds.reset_format() return self._hf_ds - - @staticmethod - def interleave_datasets(datasets: List[Any], - probabilities: Optional[List[float]] = None, - seed: Optional[int] = None): - from datasets import interleave_datasets - return interleave_datasets(datasets, probabilities, seed) diff --git a/modelscope/trainers/hooks/evaluation_hook.py b/modelscope/trainers/hooks/evaluation_hook.py index 80d8c03c..aea27f2f 100644 --- a/modelscope/trainers/hooks/evaluation_hook.py +++ b/modelscope/trainers/hooks/evaluation_hook.py @@ -32,7 +32,6 @@ class EvaluationHook(Hook): def do_evaluate(self, trainer): """Evaluate the results.""" eval_res = trainer.evaluate() - trainer.data_loader = trainer.train_dataloader for name, val in eval_res.items(): trainer.log_buffer.output[name] = val diff --git a/tests/trainers/test_trainer_with_nlp.py b/tests/trainers/test_trainer_with_nlp.py index a2d899ba..e102cd27 100644 --- a/tests/trainers/test_trainer_with_nlp.py +++ b/tests/trainers/test_trainer_with_nlp.py @@ -23,7 +23,6 @@ class TestTrainerWithNlp(unittest.TestCase): if not os.path.exists(self.tmp_dir): os.makedirs(self.tmp_dir) - # todo: Replace below scripts with MsDataset.load when the formal dataset service is ready self.dataset = MsDataset.load( 'afqmc_small', namespace='userxiaoming', split='train')