From 3aa1a70ac844ef2876eb41280d8015db0049d05c Mon Sep 17 00:00:00 2001 From: suluyan Date: Fri, 17 Jun 2022 00:36:25 +0800 Subject: [PATCH] add tests --- modelscope/models/nlp/__init__.py | 2 +- modelscope/pipelines/nlp/__init__.py | 2 +- .../pipelines/nlp/fill_mask_pipeline.py | 27 ++-- modelscope/pipelines/outputs.py | 6 + modelscope/preprocessors/nlp.py | 18 +-- requirements/nlp.txt | 2 +- tests/pipelines/test_fill_mask.py | 138 +++++++++++------- 7 files changed, 116 insertions(+), 79 deletions(-) diff --git a/modelscope/models/nlp/__init__.py b/modelscope/models/nlp/__init__.py index 2c6c6ba2..5801533b 100644 --- a/modelscope/models/nlp/__init__.py +++ b/modelscope/models/nlp/__init__.py @@ -1,4 +1,4 @@ +from .masked_language_model import * # noqa F403 from .sentence_similarity_model import * # noqa F403 from .sequence_classification_model import * # noqa F403 from .text_generation_model import * # noqa F403 -from .masked_language_model import * # noqa F403 diff --git a/modelscope/pipelines/nlp/__init__.py b/modelscope/pipelines/nlp/__init__.py index 3151b138..cf2f1c8b 100644 --- a/modelscope/pipelines/nlp/__init__.py +++ b/modelscope/pipelines/nlp/__init__.py @@ -1,4 +1,4 @@ +from .fill_mask_pipeline import * # noqa F403 from .sentence_similarity_pipeline import * # noqa F403 from .sequence_classification_pipeline import * # noqa F403 from .text_generation_pipeline import * # noqa F403 -from .fill_mask_pipeline import * # noqa F403 diff --git a/modelscope/pipelines/nlp/fill_mask_pipeline.py b/modelscope/pipelines/nlp/fill_mask_pipeline.py index aebf4e09..14b1d317 100644 --- a/modelscope/pipelines/nlp/fill_mask_pipeline.py +++ b/modelscope/pipelines/nlp/fill_mask_pipeline.py @@ -1,5 +1,6 @@ -from typing import Dict +from typing import Dict, Optional +from modelscope.models import Model from modelscope.models.nlp import MaskedLanguageModel from modelscope.preprocessors import FillMaskPreprocessor from modelscope.utils.constant import Tasks @@ -13,15 +14,23 @@ __all__ = ['FillMaskPipeline'] @PIPELINES.register_module(Tasks.fill_mask, module_name=r'veco') class FillMaskPipeline(Pipeline): - def __init__(self, model: MaskedLanguageModel, - preprocessor: FillMaskPreprocessor, **kwargs): - """use `model` and `preprocessor` to create a nlp text classification pipeline for prediction + def __init__(self, + model: MaskedLanguageModel, + preprocessor: Optional[FillMaskPreprocessor] = None, + **kwargs): + """use `model` and `preprocessor` to create a nlp fill mask pipeline for prediction Args: - model (SequenceClassificationModel): a model instance - preprocessor (SequenceClassificationPreprocessor): a preprocessor instance + model (MaskedLanguageModel): a model instance + preprocessor (FillMaskPreprocessor): a preprocessor instance """ - + sc_model = model if isinstance( + model, MaskedLanguageModel) else Model.from_pretrained(model) + if preprocessor is None: + preprocessor = FillMaskPreprocessor( + sc_model.model_dir, + first_sequence='sentence', + second_sequence=None) super().__init__(model=model, preprocessor=preprocessor, **kwargs) self.preprocessor = preprocessor self.tokenizer = preprocessor.tokenizer @@ -55,10 +64,10 @@ class FillMaskPipeline(Pipeline): pred_string = ''.join(pred_string).replace('##', '') pred_string = pred_string.split('[SEP]')[0].replace( '[CLS]', '').replace('[SEP]', '').replace('[UNK]', '') - else: ## en bert + else: # en bert pred_string = self.tokenizer.decode(ids) pred_string = pred_string.split('[SEP]')[0].replace( '[CLS]', '').replace('[SEP]', '').replace('[UNK]', '') pred_strings.append(pred_string) - return {'pred_string': pred_strings} + return {'text': pred_strings} diff --git a/modelscope/pipelines/outputs.py b/modelscope/pipelines/outputs.py index 1389abd3..b545d6eb 100644 --- a/modelscope/pipelines/outputs.py +++ b/modelscope/pipelines/outputs.py @@ -69,6 +69,12 @@ TASK_OUTPUTS = { # } Tasks.text_generation: ['text'], + # fill mask result for single sample + # { + # "text": "this is the text which masks filled by model." + # } + Tasks.fill_mask: ['text'], + # ============ audio tasks =================== # ============ multi-modal tasks =================== diff --git a/modelscope/preprocessors/nlp.py b/modelscope/preprocessors/nlp.py index 952e7b63..20c4877b 100644 --- a/modelscope/preprocessors/nlp.py +++ b/modelscope/preprocessors/nlp.py @@ -12,8 +12,7 @@ from .builder import PREPROCESSORS __all__ = [ 'Tokenize', 'SequenceClassificationPreprocessor', - 'TextGenerationPreprocessor', - 'FillMaskPreprocessor' + 'TextGenerationPreprocessor', 'FillMaskPreprocessor' ] @@ -173,8 +172,7 @@ class TextGenerationPreprocessor(Preprocessor): return {k: torch.tensor(v) for k, v in rst.items()} -@PREPROCESSORS.register_module( - Fields.nlp, module_name=r'sbert') +@PREPROCESSORS.register_module(Fields.nlp, module_name=r'sbert') class FillMaskPreprocessor(Preprocessor): def __init__(self, model_dir: str, *args, **kwargs): @@ -190,7 +188,8 @@ class FillMaskPreprocessor(Preprocessor): 'first_sequence') self.sequence_length = kwargs.pop('sequence_length', 128) - self.tokenizer = AutoTokenizer.from_pretrained(model_dir) + self.tokenizer = AutoTokenizer.from_pretrained( + model_dir, use_fast=False) @type_assert(object, str) def __call__(self, data: str) -> Dict[str, Any]: @@ -205,15 +204,11 @@ class FillMaskPreprocessor(Preprocessor): Dict[str, Any]: the preprocessed data """ import torch - + new_data = {self.first_sequence: data} # preprocess the data for the model input - rst = { - 'input_ids': [], - 'attention_mask': [], - 'token_type_ids': [] - } + rst = {'input_ids': [], 'attention_mask': [], 'token_type_ids': []} max_seq_length = self.sequence_length @@ -230,4 +225,3 @@ class FillMaskPreprocessor(Preprocessor): rst['token_type_ids'].append(feature['token_type_ids']) return {k: torch.tensor(v) for k, v in rst.items()} - diff --git a/requirements/nlp.txt b/requirements/nlp.txt index 8de83798..261b9ec5 100644 --- a/requirements/nlp.txt +++ b/requirements/nlp.txt @@ -1 +1 @@ -https://alinlp.alibaba-inc.com/pypi/sofa-1.0.1.3-py3-none-any.whl +https://alinlp.alibaba-inc.com/pypi/sofa-1.0.3-py3-none-any.whl diff --git a/tests/pipelines/test_fill_mask.py b/tests/pipelines/test_fill_mask.py index 8b021c03..b9e0defc 100644 --- a/tests/pipelines/test_fill_mask.py +++ b/tests/pipelines/test_fill_mask.py @@ -23,82 +23,110 @@ class FillMaskTest(unittest.TestCase): ori_texts = { 'zh': - f'段誉轻挥折扇，摇了摇头，说道：“你师父是你的师父，你师父可不是我的师父。' - f'你师父差得动你，你师父可差不动我。', + '段誉轻挥折扇，摇了摇头，说道：“你师父是你的师父，你师父可不是我的师父。' + '你师父差得动你，你师父可差不动我。', 'en': - f'Everything in what you call reality is really just a r' - f'eflection of your consciousness. Your whole universe is' - f'just a mirror reflection of your story.' + 'Everything in what you call reality is really just a reflection of your ' + 'consciousness. Your whole universe is just a mirror reflection of your story.' } test_inputs = { 'zh': - f'段誉轻[MASK]折扇，摇了摇[MASK]，[MASK]道：“你师父是你的[MASK][MASK]' - f'，你师父可不是[MASK]的师父。你师父差得动你，你师父可[MASK]不动我。', + '段誉轻[MASK]折扇，摇了摇[MASK]，[MASK]道：“你师父是你的[MASK][MASK]，你' + '师父可不是[MASK]的师父。你师父差得动你，你师父可[MASK]不动我。', 'en': - f'Everything in [MASK] you call reality is really [MASK] a ' - f'reflection of your [MASK]. Your whole universe is just a ' - f'mirror [MASK] of your story.' + 'Everything in [MASK] you call reality is really [MASK] a reflection of your ' + '[MASK]. Your [MASK] universe is just a mirror [MASK] of your story.' } - #def test_run(self): - # # sbert - # for language in ["zh", "en"]: - # model_dir = snapshot_download(self.model_id_sbert[language]) - # preprocessor = FillMaskPreprocessor( - # model_dir, first_sequence='sentence', second_sequence=None) - # model = MaskedLanguageModel(model_dir) - # pipeline1 = FillMaskPipeline(model, preprocessor) - # pipeline2 = pipeline( - # Tasks.fill_mask, model=model, preprocessor=preprocessor) - # ori_text = self.ori_texts[language] - # test_input = self.test_inputs[language] - # print( - # f'ori_text: {ori_text}\ninput: {test_input}\npipeline1: ' - # f'{pipeline1(test_input)}\npipeline2: {pipeline2(test_input)}' - # ) + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') + def test_run_by_direct_model_download(self): + # sbert + for language in ['zh', 'en']: + model_dir = snapshot_download(self.model_id_sbert[language]) + preprocessor = FillMaskPreprocessor( + model_dir, first_sequence='sentence', second_sequence=None) + model = MaskedLanguageModel(model_dir) + pipeline1 = FillMaskPipeline(model, preprocessor) + pipeline2 = pipeline( + Tasks.fill_mask, model=model, preprocessor=preprocessor) + ori_text = self.ori_texts[language] + test_input = self.test_inputs[language] + print( + f'\nori_text: {ori_text}\ninput: {test_input}\npipeline1: ' + f'{pipeline1(test_input)}\npipeline2: {pipeline2(test_input)}\n' + ) - ## veco - #model_dir = snapshot_download(self.model_id_veco) - #preprocessor = FillMaskPreprocessor( - # model_dir, first_sequence='sentence', second_sequence=None) - #model = MaskedLanguageModel(model_dir) - #pipeline1 = FillMaskPipeline(model, preprocessor) - #pipeline2 = pipeline( - # Tasks.fill_mask, model=model, preprocessor=preprocessor) - #for language in ["zh", "en"]: - # ori_text = self.ori_texts[language] - # test_input = self.test_inputs["zh"].replace("[MASK]", "") - # print( - # f'ori_text: {ori_text}\ninput: {test_input}\npipeline1: ' - # f'{pipeline1(test_input)}\npipeline2: {pipeline2(test_input)}' + # veco + model_dir = snapshot_download(self.model_id_veco) + preprocessor = FillMaskPreprocessor( + model_dir, first_sequence='sentence', second_sequence=None) + model = MaskedLanguageModel(model_dir) + pipeline1 = FillMaskPipeline(model, preprocessor) + pipeline2 = pipeline( + Tasks.fill_mask, model=model, preprocessor=preprocessor) + for language in ['zh', 'en']: + ori_text = self.ori_texts[language] + test_input = self.test_inputs[language].replace('[MASK]', '') + print( + f'\nori_text: {ori_text}\ninput: {test_input}\npipeline1: ' + f'{pipeline1(test_input)}\npipeline2: {pipeline2(test_input)}\n' + ) + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_with_model_from_modelhub(self): - for language in ['zh']: + # sbert + for language in ['zh', 'en']: print(self.model_id_sbert[language]) model = Model.from_pretrained(self.model_id_sbert[language]) - print('model', model.model_dir) preprocessor = FillMaskPreprocessor( model.model_dir, first_sequence='sentence', second_sequence=None) pipeline_ins = pipeline( task=Tasks.fill_mask, model=model, preprocessor=preprocessor) - print(pipeline_ins(self.test_inputs[language])) + print( + f'\nori_text: {self.ori_texts[language]}\ninput: {self.test_inputs[language]}\npipeline: ' + f'{pipeline_ins(self.test_inputs[language])}\n') + + # veco + model = Model.from_pretrained(self.model_id_veco) + preprocessor = FillMaskPreprocessor( + model.model_dir, first_sequence='sentence', second_sequence=None) + pipeline_ins = pipeline( + Tasks.fill_mask, model=model, preprocessor=preprocessor) + for language in ['zh', 'en']: + ori_text = self.ori_texts[language] + test_input = self.test_inputs[language].replace('[MASK]', '') + print(f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: ' + f'{pipeline_ins(test_input)}\n') + + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') + def test_run_with_model_name(self): + # veco + pipeline_ins = pipeline(task=Tasks.fill_mask, model=self.model_id_veco) + for language in ['zh', 'en']: + ori_text = self.ori_texts[language] + test_input = self.test_inputs[language].replace('[MASK]', '') + print(f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: ' + f'{pipeline_ins(test_input)}\n') - #def test_run_with_model_name(self): - ## veco - #pipeline_ins = pipeline( - # task=Tasks.fill_mask, model=self.model_id_veco) - #for language in ["zh", "en"]: - # input_ = self.test_inputs[language].replace("[MASK]", "") - # print(pipeline_ins(input_)) + # structBert + language = 'zh' + pipeline_ins = pipeline( + task=Tasks.fill_mask, model=self.model_id_sbert[language]) + print( + f'\nori_text: {self.ori_texts[language]}\ninput: {self.test_inputs[language]}\npipeline: ' + f'{pipeline_ins(self.test_inputs[language])}\n') - ## structBert - #for language in ["zh"]: - # pipeline_ins = pipeline( - # task=Tasks.fill_mask, model=self.model_id_sbert[language]) - # print(pipeline_ins(self_test_inputs[language])) + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') + def test_run_with_default_model(self): + pipeline_ins = pipeline(task=Tasks.fill_mask) + language = 'en' + ori_text = self.ori_texts[language] + test_input = self.test_inputs[language].replace('[MASK]', '') + print(f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: ' + f'{pipeline_ins(test_input)}\n') if __name__ == '__main__':