[to #43115042] add trainer usage doc

1. add trainer doc
2. support local configuration file for trainer
3. update nlp trainer test

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9541239

docs/source/quick_start.md

@@ -65,10 +65,6 @@ python -c "from modelscope.pipelines import pipeline;print(pipeline('word-segmen
 
 pipeline函数提供了简洁的推理接口，相关介绍和示例请参考[pipeline使用教程](tutorials/pipeline.md)
 
-## 训练
+## 训练 & 评估
 
-to be done
-
-## 评估
-
-to be done
+Trainer类提供了简洁的Finetuning和评估接口，相关介绍和示例请参考[Trainer使用教程](tutorials/trainer.md)

docs/source/tutorials/index.rst

@@ -3,3 +3,4 @@
    :caption: Tutorials
 
    pipeline.md
+   trainer.md

docs/source/tutorials/trainer.md

@@ -0,0 +1,66 @@
+# Trainer使用教程
+Modelscope提供了众多预训练模型，你可以使用其中任意一个，利用公开数据集或者私有数据集针对特定任务进行模型训练，在本篇文章中将介绍如何使用Modelscope的`Trainer`模块进行Finetuning和评估。
+
+## 环境准备
+详细步骤可以参考 [快速开始](../quick_start.md)
+
+### 准备数据集
+
+在开始Finetuning前，需要准备一个数据集用以训练和评估，详细可以参考数据集使用教程。
+
+`临时写法`，我们通过数据集接口创建一个虚假的dataset
+```python
+from datasets import Dataset
+dataset_dict = {
+    'sentence1': [
+        'This is test sentence1-1', 'This is test sentence2-1',
+        'This is test sentence3-1'
+    ],
+    'sentence2': [
+        'This is test sentence1-2', 'This is test sentence2-2',
+        'This is test sentence3-2'
+    ],
+    'label': [0, 1, 1]
+}
+train_dataset = MsDataset.from_hf_dataset(Dataset.from_dict(dataset_dict))
+eval_dataset = MsDataset.from_hf_dataset(Dataset.from_dict(dataset_dict))
+```
+### 训练
+ModelScope把所有训练相关的配置信息全部放到了模型仓库下的`configuration.json`中，因此我们只需要创建Trainer，加载配置文件，传入数据集即可完成训练。
+
+首先，通过工厂方法创建Trainer， 需要传入模型仓库路径， 训练数据集对象，评估数据集对象，训练目录
+```python
+kwargs = dict(
+    model='damo/nlp_structbert_sentiment-classification_chinese-base',
+    train_dataset=train_dataset,
+    eval_dataset=eval_dataset,
+    work_dir='work_dir')
+
+trainer = build_trainer(default_args=kwargs)
+```
+
+启动训练。
+```python
+trainer.train()
+```
+
+如果需要调整训练参数，可以在模型仓库页面下载`configuration.json`文件到本地，修改参数后，指定配置文件路径，创建trainer
+```python
+kwargs = dict(
+    model='damo/nlp_structbert_sentiment-classification_chinese-base',
+    train_dataset=train_dataset,
+    eval_dataset=eval_dataset,
+    cfg_file='你的配置文件路径'
+    work_dir='work_dir')
+
+trainer = build_trainer(default_args=kwargs)
+trainer.train()
+```
+
+
+### 评估
+训练过程中会定期使用验证集进行评估测试， Trainer模块也支持指定特定轮次保存的checkpoint路径，进行单次评估。
+```python
+eval_results = trainer.evaluate('work_dir/epoch_10.pth')
+print(eval_results)
+```

modelscope/trainers/trainer.py

@@ -93,7 +93,9 @@ class EpochBasedTrainer(BaseTrainer):
             else:
                 self.model_dir = snapshot_download(
                     model, revision=model_revision)
-            cfg_file = os.path.join(self.model_dir, ModelFile.CONFIGURATION)
+            if cfg_file is None:
+                cfg_file = os.path.join(self.model_dir,
+                                        ModelFile.CONFIGURATION)
             self.model = self.build_model()
         else:
             assert cfg_file is not None, 'Config file should not be None if model is an nn.Module class'

tests/pipelines/test_csanmt_translation.py

@@ -14,8 +14,7 @@ class TranslationTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_with_model_name(self):
-        pipeline_ins = pipeline(
-            task=Tasks.translation, model=self.model_id, model_revision='beta')
+        pipeline_ins = pipeline(task=Tasks.translation, model=self.model_id)
         print(pipeline_ins(input=self.inputs))
 
 

tests/pipelines/test_image_matting.py

@@ -46,8 +46,7 @@ class ImageMattingTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
-        img_matting = pipeline(
-            Tasks.image_matting, model=self.model_id, model_revision='beta')
+        img_matting = pipeline(Tasks.image_matting, model=self.model_id)
 
         result = img_matting('data/test/images/image_matting.png')
         cv2.imwrite('result.png', result[OutputKeys.OUTPUT_IMG])

tests/trainers/test_trainer_with_nlp.py

@@ -5,11 +5,13 @@ import tempfile
 import unittest
 
 from modelscope.hub.snapshot_download import snapshot_download
+from modelscope.metainfo import Metrics
 from modelscope.models.nlp.sbert_for_sequence_classification import \
     SbertTextClassfier
 from modelscope.msdatasets import MsDataset
 from modelscope.trainers import build_trainer
 from modelscope.utils.constant import ModelFile
+from modelscope.utils.hub import read_config
 from modelscope.utils.test_utils import test_level
 
 
@@ -73,6 +75,36 @@ class TestTrainerWithNlp(unittest.TestCase):
         for i in range(10):
             self.assertIn(f'epoch_{i+1}.pth', results_files)
 
+        eval_results = trainer.evaluate(
+            checkpoint_path=os.path.join(self.tmp_dir, 'epoch_10.pth'))
+        self.assertTrue(Metrics.accuracy in eval_results)
+
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    def test_trainer_with_user_defined_config(self):
+        model_id = 'damo/nlp_structbert_sentiment-classification_chinese-base'
+        cfg = read_config(model_id, revision='beta')
+        cfg.train.max_epochs = 20
+        cfg.train.work_dir = self.tmp_dir
+        cfg_file = os.path.join(self.tmp_dir, 'config.json')
+        cfg.dump(cfg_file)
+        kwargs = dict(
+            model=model_id,
+            train_dataset=self.dataset,
+            eval_dataset=self.dataset,
+            cfg_file=cfg_file,
+            model_revision='beta')
+
+        trainer = build_trainer(default_args=kwargs)
+        trainer.train()
+        results_files = os.listdir(self.tmp_dir)
+        self.assertIn(f'{trainer.timestamp}.log.json', results_files)
+        for i in range(20):
+            self.assertIn(f'epoch_{i+1}.pth', results_files)
+
+        eval_results = trainer.evaluate(
+            checkpoint_path=os.path.join(self.tmp_dir, 'epoch_10.pth'))
+        self.assertTrue(Metrics.accuracy in eval_results)
+
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     def test_trainer_with_model_and_args(self):
         tmp_dir = tempfile.TemporaryDirectory().name