From d4225101585db50d5958d7cdc2066c1ccfdb38a9 Mon Sep 17 00:00:00 2001 From: ly119399 Date: Thu, 30 Jun 2022 11:17:38 +0800 Subject: [PATCH] fix bug --- .../space/dialog_intent_prediction_model.py | 8 ++++---- .../models/nlp/space/dialog_modeling_model.py | 8 ++++---- .../nlp/space/model/gen_unified_transformer.py | 5 ++--- modelscope/models/nlp/space/model/generator.py | 4 +--- .../space/model/intent_unified_transformer.py | 5 ++--- .../models/nlp/space/model/model_base.py | 5 ++--- .../nlp/space/model/unified_transformer.py | 4 +--- .../models/nlp/space/modules/embedder.py | 4 +--- .../models/nlp/space/modules/feedforward.py | 4 +--- .../models/nlp/space/modules/functions.py | 4 +--- .../nlp/space/modules/multihead_attention.py | 4 +--- .../nlp/space/modules/transformer_block.py | 4 +--- .../nlp/dialog_intent_prediction_pipeline.py | 7 ++++--- .../pipelines/nlp/dialog_modeling_pipeline.py | 7 ++++--- .../preprocessors/space/fields/gen_field.py | 6 ++---- .../preprocessors/space/fields/intent_field.py | 5 ++--- modelscope/utils/nlp/space/db_ops.py | 8 -------- modelscope/utils/nlp/space/ontology.py | 13 ------------- modelscope/utils/nlp/space/utils.py | 18 ------------------ requirements/nlp.txt | 1 - .../pipelines/test_dialog_intent_prediction.py | 8 +++++--- tests/pipelines/test_dialog_modeling.py | 11 +++++------ tests/pipelines/test_nli.py | 2 +- .../pipelines/test_sentiment_classification.py | 2 +- 24 files changed, 45 insertions(+), 102 deletions(-) diff --git a/modelscope/models/nlp/space/dialog_intent_prediction_model.py b/modelscope/models/nlp/space/dialog_intent_prediction_model.py index d2e9bfa1..644af4c7 100644 --- a/modelscope/models/nlp/space/dialog_intent_prediction_model.py +++ b/modelscope/models/nlp/space/dialog_intent_prediction_model.py @@ -1,3 +1,5 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + import os from typing import Any, Dict @@ -11,20 +13,18 @@ from ...builder import MODELS from .model.generator import Generator from .model.model_base import SpaceModelBase -__all__ = ['SpaceForDialogIntentModel'] +__all__ = ['SpaceForDialogIntent'] @MODELS.register_module( Tasks.dialog_intent_prediction, module_name=Models.space) -class SpaceForDialogIntentModel(Model): +class SpaceForDialogIntent(Model): def __init__(self, model_dir: str, *args, **kwargs): """initialize the test generation model from the `model_dir` path. Args: model_dir (str): the model path. - model_cls (Optional[Any], optional): model loader, if None, use the - default loader to load model weights, by default None. """ super().__init__(model_dir, *args, **kwargs) diff --git a/modelscope/models/nlp/space/dialog_modeling_model.py b/modelscope/models/nlp/space/dialog_modeling_model.py index edb60f11..872155e2 100644 --- a/modelscope/models/nlp/space/dialog_modeling_model.py +++ b/modelscope/models/nlp/space/dialog_modeling_model.py @@ -1,3 +1,5 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + import os from typing import Any, Dict, Optional @@ -11,19 +13,17 @@ from ...builder import MODELS from .model.generator import Generator from .model.model_base import SpaceModelBase -__all__ = ['SpaceForDialogModelingModel'] +__all__ = ['SpaceForDialogModeling'] @MODELS.register_module(Tasks.dialog_modeling, module_name=Models.space) -class SpaceForDialogModelingModel(Model): +class SpaceForDialogModeling(Model): def __init__(self, model_dir: str, *args, **kwargs): """initialize the test generation model from the `model_dir` path. Args: model_dir (str): the model path. - model_cls (Optional[Any], optional): model loader, if None, use the - default loader to load model weights, by default None. """ super().__init__(model_dir, *args, **kwargs) diff --git a/modelscope/models/nlp/space/model/gen_unified_transformer.py b/modelscope/models/nlp/space/model/gen_unified_transformer.py index 0f1b1a83..c5d50cd9 100644 --- a/modelscope/models/nlp/space/model/gen_unified_transformer.py +++ b/modelscope/models/nlp/space/model/gen_unified_transformer.py @@ -1,6 +1,5 @@ -""" -IntentUnifiedTransformer -""" +# Copyright (c) Alibaba, Inc. and its affiliates. + import torch from .unified_transformer import UnifiedTransformer diff --git a/modelscope/models/nlp/space/model/generator.py b/modelscope/models/nlp/space/model/generator.py index 08e1c765..c1521e3d 100644 --- a/modelscope/models/nlp/space/model/generator.py +++ b/modelscope/models/nlp/space/model/generator.py @@ -1,6 +1,4 @@ -""" -Generator class. -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import math diff --git a/modelscope/models/nlp/space/model/intent_unified_transformer.py b/modelscope/models/nlp/space/model/intent_unified_transformer.py index b9c699d7..cae96479 100644 --- a/modelscope/models/nlp/space/model/intent_unified_transformer.py +++ b/modelscope/models/nlp/space/model/intent_unified_transformer.py @@ -1,6 +1,5 @@ -""" -IntentUnifiedTransformer -""" +# Copyright (c) Alibaba, Inc. and its affiliates. + import torch import torch.nn as nn import torch.nn.functional as F diff --git a/modelscope/models/nlp/space/model/model_base.py b/modelscope/models/nlp/space/model/model_base.py index 42496e76..7e0a6b0b 100644 --- a/modelscope/models/nlp/space/model/model_base.py +++ b/modelscope/models/nlp/space/model/model_base.py @@ -1,6 +1,5 @@ -""" -Model base -""" +# Copyright (c) Alibaba, Inc. and its affiliates. + import os import torch.nn as nn diff --git a/modelscope/models/nlp/space/model/unified_transformer.py b/modelscope/models/nlp/space/model/unified_transformer.py index 8060879d..17f9fde3 100644 --- a/modelscope/models/nlp/space/model/unified_transformer.py +++ b/modelscope/models/nlp/space/model/unified_transformer.py @@ -1,6 +1,4 @@ -""" -UnifiedTransformer -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import numpy as np import torch diff --git a/modelscope/models/nlp/space/modules/embedder.py b/modelscope/models/nlp/space/modules/embedder.py index 4fb592ef..e68ac7d3 100644 --- a/modelscope/models/nlp/space/modules/embedder.py +++ b/modelscope/models/nlp/space/modules/embedder.py @@ -1,6 +1,4 @@ -""" -Embedder class. -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import torch import torch.nn as nn diff --git a/modelscope/models/nlp/space/modules/feedforward.py b/modelscope/models/nlp/space/modules/feedforward.py index e9a5f4c7..43318eb6 100644 --- a/modelscope/models/nlp/space/modules/feedforward.py +++ b/modelscope/models/nlp/space/modules/feedforward.py @@ -1,6 +1,4 @@ -""" -FeedForward class. -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import torch import torch.nn as nn diff --git a/modelscope/models/nlp/space/modules/functions.py b/modelscope/models/nlp/space/modules/functions.py index 45c02e21..daa62bb4 100644 --- a/modelscope/models/nlp/space/modules/functions.py +++ b/modelscope/models/nlp/space/modules/functions.py @@ -1,6 +1,4 @@ -""" -Helpful functions. -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import numpy as np import torch diff --git a/modelscope/models/nlp/space/modules/multihead_attention.py b/modelscope/models/nlp/space/modules/multihead_attention.py index 209eab5e..8f981b1c 100644 --- a/modelscope/models/nlp/space/modules/multihead_attention.py +++ b/modelscope/models/nlp/space/modules/multihead_attention.py @@ -1,6 +1,4 @@ -""" -MultiheadAttention class. -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import torch import torch.nn as nn diff --git a/modelscope/models/nlp/space/modules/transformer_block.py b/modelscope/models/nlp/space/modules/transformer_block.py index 5b6c79a5..37f968d9 100644 --- a/modelscope/models/nlp/space/modules/transformer_block.py +++ b/modelscope/models/nlp/space/modules/transformer_block.py @@ -1,6 +1,4 @@ -""" -TransformerBlock class. -""" +# Copyright (c) Alibaba, Inc. and its affiliates. import torch import torch.nn as nn diff --git a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py index 4677b62e..b20c3c7c 100644 --- a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py @@ -1,7 +1,9 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + from typing import Any, Dict from ...metainfo import Pipelines -from ...models.nlp import SpaceForDialogIntentModel +from ...models.nlp import SpaceForDialogIntent from ...preprocessors import DialogIntentPredictionPreprocessor from ...utils.constant import Tasks from ..base import Pipeline @@ -15,7 +17,7 @@ __all__ = ['DialogIntentPredictionPipeline'] module_name=Pipelines.dialog_intent_prediction) class DialogIntentPredictionPipeline(Pipeline): - def __init__(self, model: SpaceForDialogIntentModel, + def __init__(self, model: SpaceForDialogIntent, preprocessor: DialogIntentPredictionPreprocessor, **kwargs): """use `model` and `preprocessor` to create a nlp text classification pipeline for prediction @@ -26,7 +28,6 @@ class DialogIntentPredictionPipeline(Pipeline): super().__init__(model=model, preprocessor=preprocessor, **kwargs) self.model = model - # self.tokenizer = preprocessor.tokenizer def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, str]: """process the prediction results diff --git a/modelscope/pipelines/nlp/dialog_modeling_pipeline.py b/modelscope/pipelines/nlp/dialog_modeling_pipeline.py index 29303d4b..76b00511 100644 --- a/modelscope/pipelines/nlp/dialog_modeling_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_modeling_pipeline.py @@ -1,7 +1,9 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + from typing import Any, Dict, Optional from ...metainfo import Pipelines -from ...models.nlp import SpaceForDialogModelingModel +from ...models.nlp import SpaceForDialogModeling from ...preprocessors import DialogModelingPreprocessor from ...utils.constant import Tasks from ..base import Pipeline, Tensor @@ -14,7 +16,7 @@ __all__ = ['DialogModelingPipeline'] Tasks.dialog_modeling, module_name=Pipelines.dialog_modeling) class DialogModelingPipeline(Pipeline): - def __init__(self, model: SpaceForDialogModelingModel, + def __init__(self, model: SpaceForDialogModeling, preprocessor: DialogModelingPreprocessor, **kwargs): """use `model` and `preprocessor` to create a nlp text classification pipeline for prediction @@ -40,7 +42,6 @@ class DialogModelingPipeline(Pipeline): inputs['resp']) assert len(sys_rsp) > 2 sys_rsp = sys_rsp[1:len(sys_rsp) - 1] - # sys_rsp = self.preprocessor.text_field.tokenizer. inputs['sys'] = sys_rsp diff --git a/modelscope/preprocessors/space/fields/gen_field.py b/modelscope/preprocessors/space/fields/gen_field.py index fa037145..28928029 100644 --- a/modelscope/preprocessors/space/fields/gen_field.py +++ b/modelscope/preprocessors/space/fields/gen_field.py @@ -1,6 +1,5 @@ -""" -Field class -""" +# Copyright (c) Alibaba, Inc. and its affiliates. + import os import random from collections import OrderedDict @@ -8,7 +7,6 @@ from itertools import chain import numpy as np -from ....utils.constant import ModelFile from ....utils.nlp.space import ontology, utils from ....utils.nlp.space.db_ops import MultiWozDB from ....utils.nlp.space.utils import list2np diff --git a/modelscope/preprocessors/space/fields/intent_field.py b/modelscope/preprocessors/space/fields/intent_field.py index 15bd20b6..d7f69eec 100644 --- a/modelscope/preprocessors/space/fields/intent_field.py +++ b/modelscope/preprocessors/space/fields/intent_field.py @@ -1,6 +1,5 @@ -""" -Intent Field class -""" +# Copyright (c) Alibaba, Inc. and its affiliates. + import glob import multiprocessing import os diff --git a/modelscope/utils/nlp/space/db_ops.py b/modelscope/utils/nlp/space/db_ops.py index 10c3aab7..880b018b 100644 --- a/modelscope/utils/nlp/space/db_ops.py +++ b/modelscope/utils/nlp/space/db_ops.py @@ -308,14 +308,6 @@ if __name__ == '__main__': 'attraction': 5, 'train': 1, } - # for ent in res: - # if reidx.get(domain): - # report.append(ent[reidx[domain]]) - # for ent in res: - # if 'name' in ent: - # report.append(ent['name']) - # if 'trainid' in ent: - # report.append(ent['trainid']) print(constraints) print(res) print('count:', len(res), '\nnames:', report) diff --git a/modelscope/utils/nlp/space/ontology.py b/modelscope/utils/nlp/space/ontology.py index 4f27168a..99b084bb 100644 --- a/modelscope/utils/nlp/space/ontology.py +++ b/modelscope/utils/nlp/space/ontology.py @@ -123,19 +123,6 @@ dialog_act_all_slots = all_slots + ['choice', 'open'] # no need of this, just covert slot to [slot] e.g. pricerange -> [pricerange] slot_name_to_slot_token = {} -# special slot tokens in responses -# not use at the momoent -slot_name_to_value_token = { - # 'entrance fee': '[value_price]', - # 'pricerange': '[value_price]', - # 'arriveby': '[value_time]', - # 'leaveat': '[value_time]', - # 'departure': '[value_place]', - # 'destination': '[value_place]', - # 'stay': 'count', - # 'people': 'count' -} - # eos tokens definition eos_tokens = { 'user': '', diff --git a/modelscope/utils/nlp/space/utils.py b/modelscope/utils/nlp/space/utils.py index ba956b7d..ef38684a 100644 --- a/modelscope/utils/nlp/space/utils.py +++ b/modelscope/utils/nlp/space/utils.py @@ -53,16 +53,9 @@ def clean_replace(s, r, t, forward=True, backward=False): return s, -1 return s[:idx] + t + s[idx_r:], idx_r - # source, replace, target = s, r, t - # count = 0 sidx = 0 while sidx != -1: s, sidx = clean_replace_single(s, r, t, forward, backward, sidx) - # count += 1 - # print(s, sidx) - # if count == 20: - # print(source, '\n', replace, '\n', target) - # quit() return s @@ -193,14 +186,3 @@ class MultiWOZVocab(object): return self._idx2word[idx] else: return self._idx2word[idx] + '(o)' - - # def sentence_decode(self, index_list, eos=None, indicate_oov=False): - # l = [self.decode(_, indicate_oov) for _ in index_list] - # if not eos or eos not in l: - # return ' '.join(l) - # else: - # idx = l.index(eos) - # return ' '.join(l[:idx]) - # - # def nl_decode(self, l, eos=None): - # return [self.sentence_decode(_, eos) + '\n' for _ in l] diff --git a/requirements/nlp.txt b/requirements/nlp.txt index 84a57b5c..75c7637f 100644 --- a/requirements/nlp.txt +++ b/requirements/nlp.txt @@ -1,4 +1,3 @@ https://alinlp.alibaba-inc.com/pypi/sofa-1.0.3-py3-none-any.whl https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz spacy>=2.3.5 -# python -m spacy download en_core_web_sm diff --git a/tests/pipelines/test_dialog_intent_prediction.py b/tests/pipelines/test_dialog_intent_prediction.py index ae3a9bf1..051f979b 100644 --- a/tests/pipelines/test_dialog_intent_prediction.py +++ b/tests/pipelines/test_dialog_intent_prediction.py @@ -3,10 +3,11 @@ import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.models import Model -from modelscope.models.nlp import SpaceForDialogIntentModel +from modelscope.models.nlp import SpaceForDialogIntent from modelscope.pipelines import DialogIntentPredictionPipeline, pipeline from modelscope.preprocessors import DialogIntentPredictionPreprocessor from modelscope.utils.constant import Tasks +from modelscope.utils.test_utils import test_level class DialogIntentPredictionTest(unittest.TestCase): @@ -16,11 +17,11 @@ class DialogIntentPredictionTest(unittest.TestCase): 'I still have not received my new card, I ordered over a week ago.' ] - @unittest.skip('test with snapshot_download') + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run(self): cache_path = snapshot_download(self.model_id) preprocessor = DialogIntentPredictionPreprocessor(model_dir=cache_path) - model = SpaceForDialogIntentModel( + model = SpaceForDialogIntent( model_dir=cache_path, text_field=preprocessor.text_field, config=preprocessor.config) @@ -37,6 +38,7 @@ class DialogIntentPredictionTest(unittest.TestCase): for my_pipeline, item in list(zip(pipelines, self.test_case)): print(my_pipeline(item)) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_from_modelhub(self): model = Model.from_pretrained(self.model_id) preprocessor = DialogIntentPredictionPreprocessor( diff --git a/tests/pipelines/test_dialog_modeling.py b/tests/pipelines/test_dialog_modeling.py index 79644bc5..cd17502e 100644 --- a/tests/pipelines/test_dialog_modeling.py +++ b/tests/pipelines/test_dialog_modeling.py @@ -1,15 +1,13 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -import os -import os.path as osp -import tempfile import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.models import Model -from modelscope.models.nlp import SpaceForDialogModelingModel +from modelscope.models.nlp import SpaceForDialogModeling from modelscope.pipelines import DialogModelingPipeline, pipeline from modelscope.preprocessors import DialogModelingPreprocessor from modelscope.utils.constant import Tasks +from modelscope.utils.test_utils import test_level class DialogModelingTest(unittest.TestCase): @@ -91,13 +89,13 @@ class DialogModelingTest(unittest.TestCase): } } - @unittest.skip('test with snapshot_download') + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run(self): cache_path = snapshot_download(self.model_id) preprocessor = DialogModelingPreprocessor(model_dir=cache_path) - model = SpaceForDialogModelingModel( + model = SpaceForDialogModeling( model_dir=cache_path, text_field=preprocessor.text_field, config=preprocessor.config) @@ -120,6 +118,7 @@ class DialogModelingTest(unittest.TestCase): }) print('sys : {}'.format(result['sys'])) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_from_modelhub(self): model = Model.from_pretrained(self.model_id) preprocessor = DialogModelingPreprocessor(model_dir=model.model_dir) diff --git a/tests/pipelines/test_nli.py b/tests/pipelines/test_nli.py index 0c8da8b4..b78acfdf 100644 --- a/tests/pipelines/test_nli.py +++ b/tests/pipelines/test_nli.py @@ -37,7 +37,7 @@ class NLITest(unittest.TestCase): task=Tasks.nli, model=model, preprocessor=tokenizer) print(pipeline_ins(input=(self.sentence1, self.sentence2))) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_with_model_name(self): pipeline_ins = pipeline(task=Tasks.nli, model=self.model_id) print(pipeline_ins(input=(self.sentence1, self.sentence2))) diff --git a/tests/pipelines/test_sentiment_classification.py b/tests/pipelines/test_sentiment_classification.py index 0ba22d5c..829c0f7d 100644 --- a/tests/pipelines/test_sentiment_classification.py +++ b/tests/pipelines/test_sentiment_classification.py @@ -42,7 +42,7 @@ class SentimentClassificationTest(unittest.TestCase): preprocessor=tokenizer) print(pipeline_ins(input=self.sentence1)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_with_model_name(self): pipeline_ins = pipeline( task=Tasks.sentiment_classification, model=self.model_id)