From 1cc1b3c63711d8542f2735064c886010eeb78ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=99=BA=E4=B8=9E?= Date: Mon, 27 Jun 2022 14:28:05 +0800 Subject: [PATCH] revise based on comment except chinease comment --- .../nlp/bert_for_sequence_classification.py | 4 ++-- .../models/nlp/sbert_for_sentence_similarity.py | 4 ++-- .../nlp/sbert_for_sentiment_classification.py | 4 ++-- .../models/nlp/sbert_for_token_classification.py | 4 ++-- .../nlp/space/dialog_intent_prediction_model.py | 14 +++++++------- .../models/nlp/space/dialog_modeling_model.py | 14 +++++++------- modelscope/models/nlp/space/model/model_base.py | 15 +++++++++------ .../models/nlp/space/model/unified_transformer.py | 4 ++-- .../nlp/dialog_intent_prediction_pipeline.py | 4 ++-- .../pipelines/nlp/dialog_modeling_pipeline.py | 8 ++++---- .../dialog_intent_prediction_preprocessor.py | 4 ++-- .../space/dialog_modeling_preprocessor.py | 4 ++-- .../preprocessors/space/fields/gen_field.py | 1 + .../nlp/space}/__init__.py | 0 .../nlp/space/metrics/__init__.py | 0 .../nlp/space/metrics/metrics_tracker.py | 0 modelscope/trainers/nlp/space/trainer/__init__.py | 0 .../nlp/space/trainer/gen_trainer.py} | 0 .../nlp/space/trainer/intent_trainer.py} | 0 tests/pipelines/test_dialog_intent_prediction.py | 4 ++-- tests/pipelines/test_dialog_modeling.py | 4 ++-- 21 files changed, 48 insertions(+), 44 deletions(-) rename modelscope/{models/nlp/space/application => trainers/nlp/space}/__init__.py (100%) rename modelscope/{models => trainers}/nlp/space/metrics/__init__.py (100%) rename modelscope/{models => trainers}/nlp/space/metrics/metrics_tracker.py (100%) create mode 100644 modelscope/trainers/nlp/space/trainer/__init__.py rename modelscope/{models/nlp/space/application/gen_app.py => trainers/nlp/space/trainer/gen_trainer.py} (100%) rename modelscope/{models/nlp/space/application/intent_app.py => trainers/nlp/space/trainer/intent_trainer.py} (100%) diff --git a/modelscope/models/nlp/bert_for_sequence_classification.py b/modelscope/models/nlp/bert_for_sequence_classification.py index 7d85fa28..6eb27f03 100644 --- a/modelscope/models/nlp/bert_for_sequence_classification.py +++ b/modelscope/models/nlp/bert_for_sequence_classification.py @@ -4,8 +4,8 @@ from typing import Any, Dict import json import numpy as np -from modelscope.metainfo import Models -from modelscope.utils.constant import Tasks +from ...metainfo import Models +from ...utils.constant import Tasks from ..base import Model from ..builder import MODELS diff --git a/modelscope/models/nlp/sbert_for_sentence_similarity.py b/modelscope/models/nlp/sbert_for_sentence_similarity.py index 25c38a2e..5fa487e5 100644 --- a/modelscope/models/nlp/sbert_for_sentence_similarity.py +++ b/modelscope/models/nlp/sbert_for_sentence_similarity.py @@ -1,5 +1,5 @@ -from modelscope.metainfo import Models -from modelscope.utils.constant import Tasks +from ...metainfo import Models +from ...utils.constant import Tasks from ..builder import MODELS from .sbert_for_sequence_classification import \ SbertForSequenceClassificationBase diff --git a/modelscope/models/nlp/sbert_for_sentiment_classification.py b/modelscope/models/nlp/sbert_for_sentiment_classification.py index 72fb92f0..00ec8b73 100644 --- a/modelscope/models/nlp/sbert_for_sentiment_classification.py +++ b/modelscope/models/nlp/sbert_for_sentiment_classification.py @@ -1,5 +1,5 @@ -from modelscope.metainfo import Models -from modelscope.utils.constant import Tasks +from ...metainfo import Models +from ...utils.constant import Tasks from ..builder import MODELS from .sbert_for_sequence_classification import \ SbertForSequenceClassificationBase diff --git a/modelscope/models/nlp/sbert_for_token_classification.py b/modelscope/models/nlp/sbert_for_token_classification.py index fd175033..a23002ee 100644 --- a/modelscope/models/nlp/sbert_for_token_classification.py +++ b/modelscope/models/nlp/sbert_for_token_classification.py @@ -3,8 +3,8 @@ from typing import Any, Dict, Union import numpy as np import torch -from modelscope.metainfo import Models -from modelscope.utils.constant import Tasks +from ...metainfo import Models +from ...utils.constant import Tasks from ..base import Model, Tensor from ..builder import MODELS diff --git a/modelscope/models/nlp/space/dialog_intent_prediction_model.py b/modelscope/models/nlp/space/dialog_intent_prediction_model.py index a6bd1d27..74e4e9e7 100644 --- a/modelscope/models/nlp/space/dialog_intent_prediction_model.py +++ b/modelscope/models/nlp/space/dialog_intent_prediction_model.py @@ -2,19 +2,19 @@ import os from typing import Any, Dict from ....preprocessors.space.fields.intent_field import IntentBPETextField +from ....trainers.nlp.space.trainer.intent_trainer import IntentTrainer from ....utils.config import Config -from ....utils.constant import Tasks +from ....utils.constant import ModelFile, Tasks from ...base import Model, Tensor from ...builder import MODELS -from .application.intent_app import IntentTrainer from .model.generator import Generator -from .model.model_base import ModelBase +from .model.model_base import SpaceModelBase -__all__ = ['DialogIntentModel'] +__all__ = ['SpaceForDialogIntentModel'] @MODELS.register_module(Tasks.dialog_intent_prediction, module_name=r'space') -class DialogIntentModel(Model): +class SpaceForDialogIntentModel(Model): def __init__(self, model_dir: str, *args, **kwargs): """initialize the test generation model from the `model_dir` path. @@ -30,13 +30,13 @@ class DialogIntentModel(Model): self.config = kwargs.pop( 'config', Config.from_file( - os.path.join(self.model_dir, 'configuration.json'))) + os.path.join(self.model_dir, ModelFile.CONFIGURATION))) self.text_field = kwargs.pop( 'text_field', IntentBPETextField(self.model_dir, config=self.config)) self.generator = Generator.create(self.config, reader=self.text_field) - self.model = ModelBase.create( + self.model = SpaceModelBase.create( model_dir=model_dir, config=self.config, reader=self.text_field, diff --git a/modelscope/models/nlp/space/dialog_modeling_model.py b/modelscope/models/nlp/space/dialog_modeling_model.py index ad8212c0..e11ef9fd 100644 --- a/modelscope/models/nlp/space/dialog_modeling_model.py +++ b/modelscope/models/nlp/space/dialog_modeling_model.py @@ -2,19 +2,19 @@ import os from typing import Any, Dict, Optional from ....preprocessors.space.fields.gen_field import MultiWOZBPETextField +from ....trainers.nlp.space.trainer.gen_trainer import MultiWOZTrainer from ....utils.config import Config -from ....utils.constant import Tasks +from ....utils.constant import ModelFile, Tasks from ...base import Model, Tensor from ...builder import MODELS -from .application.gen_app import MultiWOZTrainer from .model.generator import Generator -from .model.model_base import ModelBase +from .model.model_base import SpaceModelBase -__all__ = ['DialogModelingModel'] +__all__ = ['SpaceForDialogModelingModel'] @MODELS.register_module(Tasks.dialog_modeling, module_name=r'space') -class DialogModelingModel(Model): +class SpaceForDialogModelingModel(Model): def __init__(self, model_dir: str, *args, **kwargs): """initialize the test generation model from the `model_dir` path. @@ -30,12 +30,12 @@ class DialogModelingModel(Model): self.config = kwargs.pop( 'config', Config.from_file( - os.path.join(self.model_dir, 'configuration.json'))) + os.path.join(self.model_dir, ModelFile.CONFIGURATION))) self.text_field = kwargs.pop( 'text_field', MultiWOZBPETextField(self.model_dir, config=self.config)) self.generator = Generator.create(self.config, reader=self.text_field) - self.model = ModelBase.create( + self.model = SpaceModelBase.create( model_dir=model_dir, config=self.config, reader=self.text_field, diff --git a/modelscope/models/nlp/space/model/model_base.py b/modelscope/models/nlp/space/model/model_base.py index cdd355a5..42496e76 100644 --- a/modelscope/models/nlp/space/model/model_base.py +++ b/modelscope/models/nlp/space/model/model_base.py @@ -5,8 +5,10 @@ import os import torch.nn as nn +from .....utils.constant import ModelFile -class ModelBase(nn.Module): + +class SpaceModelBase(nn.Module): """ Basic model wrapper for static graph and dygrpah. """ @@ -14,21 +16,22 @@ class ModelBase(nn.Module): @classmethod def register(cls, name): - ModelBase._registry[name] = cls + SpaceModelBase._registry[name] = cls return @staticmethod def by_name(name): - return ModelBase._registry[name] + return SpaceModelBase._registry[name] @staticmethod def create(model_dir, config, *args, **kwargs): - model_cls = ModelBase.by_name(config.Model.model) + model_cls = SpaceModelBase.by_name(config.Model.model) return model_cls(model_dir, config, *args, **kwargs) def __init__(self, model_dir, config): - super(ModelBase, self).__init__() - self.init_checkpoint = os.path.join(model_dir, 'pytorch_model.bin') + super(SpaceModelBase, self).__init__() + self.init_checkpoint = os.path.join(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) self.abandon_label = config.Dataset.abandon_label self.use_gpu = config.use_gpu self.gpu = config.Trainer.gpu diff --git a/modelscope/models/nlp/space/model/unified_transformer.py b/modelscope/models/nlp/space/model/unified_transformer.py index 2636553d..15a18056 100644 --- a/modelscope/models/nlp/space/model/unified_transformer.py +++ b/modelscope/models/nlp/space/model/unified_transformer.py @@ -9,10 +9,10 @@ import torch.nn.functional as F from ..modules.embedder import Embedder from ..modules.transformer_block import TransformerBlock -from .model_base import ModelBase +from .model_base import SpaceModelBase -class UnifiedTransformer(ModelBase): +class UnifiedTransformer(SpaceModelBase): """ Implement unified transformer. """ diff --git a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py index 3fd38641..4677b62e 100644 --- a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py @@ -1,7 +1,7 @@ from typing import Any, Dict from ...metainfo import Pipelines -from ...models.nlp import DialogIntentModel +from ...models.nlp import SpaceForDialogIntentModel from ...preprocessors import DialogIntentPredictionPreprocessor from ...utils.constant import Tasks from ..base import Pipeline @@ -15,7 +15,7 @@ __all__ = ['DialogIntentPredictionPipeline'] module_name=Pipelines.dialog_intent_prediction) class DialogIntentPredictionPipeline(Pipeline): - def __init__(self, model: DialogIntentModel, + def __init__(self, model: SpaceForDialogIntentModel, preprocessor: DialogIntentPredictionPreprocessor, **kwargs): """use `model` and `preprocessor` to create a nlp text classification pipeline for prediction diff --git a/modelscope/pipelines/nlp/dialog_modeling_pipeline.py b/modelscope/pipelines/nlp/dialog_modeling_pipeline.py index 778284de..29303d4b 100644 --- a/modelscope/pipelines/nlp/dialog_modeling_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_modeling_pipeline.py @@ -1,9 +1,9 @@ from typing import Any, Dict, Optional -from modelscope.models.nlp import DialogModelingModel -from modelscope.preprocessors import DialogModelingPreprocessor -from modelscope.utils.constant import Tasks from ...metainfo import Pipelines +from ...models.nlp import SpaceForDialogModelingModel +from ...preprocessors import DialogModelingPreprocessor +from ...utils.constant import Tasks from ..base import Pipeline, Tensor from ..builder import PIPELINES @@ -14,7 +14,7 @@ __all__ = ['DialogModelingPipeline'] Tasks.dialog_modeling, module_name=Pipelines.dialog_modeling) class DialogModelingPipeline(Pipeline): - def __init__(self, model: DialogModelingModel, + def __init__(self, model: SpaceForDialogModelingModel, preprocessor: DialogModelingPreprocessor, **kwargs): """use `model` and `preprocessor` to create a nlp text classification pipeline for prediction diff --git a/modelscope/preprocessors/space/dialog_intent_prediction_preprocessor.py b/modelscope/preprocessors/space/dialog_intent_prediction_preprocessor.py index 733abf24..4b46b044 100644 --- a/modelscope/preprocessors/space/dialog_intent_prediction_preprocessor.py +++ b/modelscope/preprocessors/space/dialog_intent_prediction_preprocessor.py @@ -4,7 +4,7 @@ import os from typing import Any, Dict from ...utils.config import Config -from ...utils.constant import Fields +from ...utils.constant import Fields, ModelFile from ...utils.type_assert import type_assert from ..base import Preprocessor from ..builder import PREPROCESSORS @@ -26,7 +26,7 @@ class DialogIntentPredictionPreprocessor(Preprocessor): self.model_dir: str = model_dir self.config = Config.from_file( - os.path.join(self.model_dir, 'configuration.json')) + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) self.text_field = IntentBPETextField( self.model_dir, config=self.config) diff --git a/modelscope/preprocessors/space/dialog_modeling_preprocessor.py b/modelscope/preprocessors/space/dialog_modeling_preprocessor.py index b0758b40..d5e02c4a 100644 --- a/modelscope/preprocessors/space/dialog_modeling_preprocessor.py +++ b/modelscope/preprocessors/space/dialog_modeling_preprocessor.py @@ -4,7 +4,7 @@ import os from typing import Any, Dict from ...utils.config import Config -from ...utils.constant import Fields +from ...utils.constant import Fields, ModelFile from ...utils.type_assert import type_assert from ..base import Preprocessor from ..builder import PREPROCESSORS @@ -26,7 +26,7 @@ class DialogModelingPreprocessor(Preprocessor): self.model_dir: str = model_dir self.config = Config.from_file( - os.path.join(self.model_dir, 'configuration.json')) + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) self.text_field = MultiWOZBPETextField( self.model_dir, config=self.config) diff --git a/modelscope/preprocessors/space/fields/gen_field.py b/modelscope/preprocessors/space/fields/gen_field.py index 49a30e8f..410aea7e 100644 --- a/modelscope/preprocessors/space/fields/gen_field.py +++ b/modelscope/preprocessors/space/fields/gen_field.py @@ -8,6 +8,7 @@ from itertools import chain import numpy as np +from ....utils.constant import ModelFile from ....utils.nlp.space import ontology, utils from ....utils.nlp.space.db_ops import MultiWozDB from ....utils.nlp.space.utils import list2np diff --git a/modelscope/models/nlp/space/application/__init__.py b/modelscope/trainers/nlp/space/__init__.py similarity index 100% rename from modelscope/models/nlp/space/application/__init__.py rename to modelscope/trainers/nlp/space/__init__.py diff --git a/modelscope/models/nlp/space/metrics/__init__.py b/modelscope/trainers/nlp/space/metrics/__init__.py similarity index 100% rename from modelscope/models/nlp/space/metrics/__init__.py rename to modelscope/trainers/nlp/space/metrics/__init__.py diff --git a/modelscope/models/nlp/space/metrics/metrics_tracker.py b/modelscope/trainers/nlp/space/metrics/metrics_tracker.py similarity index 100% rename from modelscope/models/nlp/space/metrics/metrics_tracker.py rename to modelscope/trainers/nlp/space/metrics/metrics_tracker.py diff --git a/modelscope/trainers/nlp/space/trainer/__init__.py b/modelscope/trainers/nlp/space/trainer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modelscope/models/nlp/space/application/gen_app.py b/modelscope/trainers/nlp/space/trainer/gen_trainer.py similarity index 100% rename from modelscope/models/nlp/space/application/gen_app.py rename to modelscope/trainers/nlp/space/trainer/gen_trainer.py diff --git a/modelscope/models/nlp/space/application/intent_app.py b/modelscope/trainers/nlp/space/trainer/intent_trainer.py similarity index 100% rename from modelscope/models/nlp/space/application/intent_app.py rename to modelscope/trainers/nlp/space/trainer/intent_trainer.py diff --git a/tests/pipelines/test_dialog_intent_prediction.py b/tests/pipelines/test_dialog_intent_prediction.py index 97cdbb3d..ae3a9bf1 100644 --- a/tests/pipelines/test_dialog_intent_prediction.py +++ b/tests/pipelines/test_dialog_intent_prediction.py @@ -3,7 +3,7 @@ import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.models import Model -from modelscope.models.nlp import DialogIntentModel +from modelscope.models.nlp import SpaceForDialogIntentModel from modelscope.pipelines import DialogIntentPredictionPipeline, pipeline from modelscope.preprocessors import DialogIntentPredictionPreprocessor from modelscope.utils.constant import Tasks @@ -20,7 +20,7 @@ class DialogIntentPredictionTest(unittest.TestCase): def test_run(self): cache_path = snapshot_download(self.model_id) preprocessor = DialogIntentPredictionPreprocessor(model_dir=cache_path) - model = DialogIntentModel( + model = SpaceForDialogIntentModel( model_dir=cache_path, text_field=preprocessor.text_field, config=preprocessor.config) diff --git a/tests/pipelines/test_dialog_modeling.py b/tests/pipelines/test_dialog_modeling.py index f606ba49..79644bc5 100644 --- a/tests/pipelines/test_dialog_modeling.py +++ b/tests/pipelines/test_dialog_modeling.py @@ -6,7 +6,7 @@ import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.models import Model -from modelscope.models.nlp import DialogModelingModel +from modelscope.models.nlp import SpaceForDialogModelingModel from modelscope.pipelines import DialogModelingPipeline, pipeline from modelscope.preprocessors import DialogModelingPreprocessor from modelscope.utils.constant import Tasks @@ -97,7 +97,7 @@ class DialogModelingTest(unittest.TestCase): cache_path = snapshot_download(self.model_id) preprocessor = DialogModelingPreprocessor(model_dir=cache_path) - model = DialogModelingModel( + model = SpaceForDialogModelingModel( model_dir=cache_path, text_field=preprocessor.text_field, config=preprocessor.config)