Browse Source

[to #42322933] Refine the nlp examples of finetuning

1. Refine the nlp finetune examples
2. Remove some useless code
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9631158
master
yuze.zyz 3 years ago
parent
commit
064f1041a9
3 changed files with 0 additions and 9 deletions
  1. +0
    -7
      modelscope/msdatasets/ms_dataset.py
  2. +0
    -1
      modelscope/trainers/hooks/evaluation_hook.py
  3. +0
    -1
      tests/trainers/test_trainer_with_nlp.py

+ 0
- 7
modelscope/msdatasets/ms_dataset.py View File

@@ -517,10 +517,3 @@ class MsDataset:
def to_hf_dataset(self) -> Dataset: def to_hf_dataset(self) -> Dataset:
self._hf_ds.reset_format() self._hf_ds.reset_format()
return self._hf_ds return self._hf_ds

@staticmethod
def interleave_datasets(datasets: List[Any],
probabilities: Optional[List[float]] = None,
seed: Optional[int] = None):
from datasets import interleave_datasets
return interleave_datasets(datasets, probabilities, seed)

+ 0
- 1
modelscope/trainers/hooks/evaluation_hook.py View File

@@ -32,7 +32,6 @@ class EvaluationHook(Hook):
def do_evaluate(self, trainer): def do_evaluate(self, trainer):
"""Evaluate the results.""" """Evaluate the results."""
eval_res = trainer.evaluate() eval_res = trainer.evaluate()
trainer.data_loader = trainer.train_dataloader
for name, val in eval_res.items(): for name, val in eval_res.items():
trainer.log_buffer.output[name] = val trainer.log_buffer.output[name] = val




+ 0
- 1
tests/trainers/test_trainer_with_nlp.py View File

@@ -23,7 +23,6 @@ class TestTrainerWithNlp(unittest.TestCase):
if not os.path.exists(self.tmp_dir): if not os.path.exists(self.tmp_dir):
os.makedirs(self.tmp_dir) os.makedirs(self.tmp_dir)


# todo: Replace below scripts with MsDataset.load when the formal dataset service is ready
self.dataset = MsDataset.load( self.dataset = MsDataset.load(
'afqmc_small', namespace='userxiaoming', split='train') 'afqmc_small', namespace='userxiaoming', split='train')




Loading…
Cancel
Save