From 8246174104b486bfd10bb85120cb4778a0f9b6da Mon Sep 17 00:00:00 2001 From: ly119399 Date: Thu, 4 Aug 2022 09:10:15 +0800 Subject: [PATCH] [to #43627720] pipeline task_oriented_conversation Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9608407 --- modelscope/metainfo.py | 2 +- .../nlp/space/space_for_dialog_modeling.py | 7 +-- modelscope/outputs.py | 2 +- modelscope/pipelines/builder.py | 4 +- modelscope/pipelines/nlp/__init__.py | 5 +- ...=> task_oriented_conversation_pipeline.py} | 7 +-- modelscope/utils/constant.py | 2 +- ....py => test_task_oriented_conversation.py} | 52 +++++++++++-------- 8 files changed, 45 insertions(+), 36 deletions(-) rename modelscope/pipelines/nlp/{dialog_modeling_pipeline.py => task_oriented_conversation_pipeline.py} (90%) rename tests/pipelines/{test_dialog_modeling.py => test_task_oriented_conversation.py} (79%) diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index e0326baa..d889f5cf 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -112,7 +112,7 @@ class Pipelines(object): csanmt_translation = 'csanmt-translation' nli = 'nli' dialog_intent_prediction = 'dialog-intent-prediction' - dialog_modeling = 'dialog-modeling' + task_oriented_conversation = 'task-oriented-conversation' dialog_state_tracking = 'dialog-state-tracking' zero_shot_classification = 'zero-shot-classification' text_error_correction = 'text-error-correction' diff --git a/modelscope/models/nlp/space/space_for_dialog_modeling.py b/modelscope/models/nlp/space/space_for_dialog_modeling.py index 8b9ed8b3..6ffc2254 100644 --- a/modelscope/models/nlp/space/space_for_dialog_modeling.py +++ b/modelscope/models/nlp/space/space_for_dialog_modeling.py @@ -15,7 +15,8 @@ from modelscope.utils.constant import ModelFile, Tasks __all__ = ['SpaceForDialogModeling'] -@MODELS.register_module(Tasks.dialog_modeling, module_name=Models.space) +@MODELS.register_module( + Tasks.task_oriented_conversation, module_name=Models.space) class SpaceForDialogModeling(TorchModel): def __init__(self, model_dir: str, *args, **kwargs): @@ -33,8 +34,8 @@ class SpaceForDialogModeling(TorchModel): Config.from_file( os.path.join(self.model_dir, ModelFile.CONFIGURATION))) - import torch - self.config.use_gpu = self.config.use_gpu and torch.cuda.is_available() + self.config.use_gpu = True if 'device' not in kwargs or kwargs[ + 'device'] == 'gpu' else False self.text_field = kwargs.pop( 'text_field', diff --git a/modelscope/outputs.py b/modelscope/outputs.py index a82f6ed5..111e90c5 100644 --- a/modelscope/outputs.py +++ b/modelscope/outputs.py @@ -326,7 +326,7 @@ TASK_OUTPUTS = { # (Deprecated) dialog modeling prediction result for single sample # sys : ['you', 'are', 'welcome', '.', 'have', 'a', 'great', 'day', '!'] - Tasks.dialog_modeling: [OutputKeys.RESPONSE], + Tasks.task_oriented_conversation: [OutputKeys.RESPONSE], # (Deprecated) dialog state tracking result for single sample # { diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 6ef21752..7c0a408f 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -51,8 +51,8 @@ DEFAULT_MODEL_FOR_PIPELINE = { Tasks.dialog_intent_prediction: (Pipelines.dialog_intent_prediction, 'damo/nlp_space_dialog-intent-prediction'), - Tasks.dialog_modeling: (Pipelines.dialog_modeling, - 'damo/nlp_space_dialog-modeling'), + Tasks.task_oriented_conversation: (Pipelines.task_oriented_conversation, + 'damo/nlp_space_dialog-modeling'), Tasks.dialog_state_tracking: (Pipelines.dialog_state_tracking, 'damo/nlp_space_dialog-state-tracking'), Tasks.text_error_correction: diff --git a/modelscope/pipelines/nlp/__init__.py b/modelscope/pipelines/nlp/__init__.py index 1111f0d3..fb158775 100644 --- a/modelscope/pipelines/nlp/__init__.py +++ b/modelscope/pipelines/nlp/__init__.py @@ -5,7 +5,7 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .dialog_intent_prediction_pipeline import DialogIntentPredictionPipeline - from .dialog_modeling_pipeline import DialogModelingPipeline + from .task_oriented_conversation_pipeline import TaskOrientedConversationPipeline from .dialog_state_tracking_pipeline import DialogStateTrackingPipeline from .fill_mask_pipeline import FillMaskPipeline from .named_entity_recognition_pipeline import NamedEntityRecognitionPipeline @@ -24,7 +24,8 @@ else: _import_structure = { 'dialog_intent_prediction_pipeline': ['DialogIntentPredictionPipeline'], - 'dialog_modeling_pipeline': ['DialogModelingPipeline'], + 'task_oriented_conversation_pipeline': + ['TaskOrientedConversationPipeline'], 'dialog_state_tracking_pipeline': ['DialogStateTrackingPipeline'], 'fill_mask_pipeline': ['FillMaskPipeline'], 'single_sentence_classification_pipeline': diff --git a/modelscope/pipelines/nlp/dialog_modeling_pipeline.py b/modelscope/pipelines/nlp/task_oriented_conversation_pipeline.py similarity index 90% rename from modelscope/pipelines/nlp/dialog_modeling_pipeline.py rename to modelscope/pipelines/nlp/task_oriented_conversation_pipeline.py index 7cbfa5bf..e946596f 100644 --- a/modelscope/pipelines/nlp/dialog_modeling_pipeline.py +++ b/modelscope/pipelines/nlp/task_oriented_conversation_pipeline.py @@ -11,12 +11,13 @@ from modelscope.pipelines.builder import PIPELINES from modelscope.preprocessors import DialogModelingPreprocessor from modelscope.utils.constant import Tasks -__all__ = ['DialogModelingPipeline'] +__all__ = ['TaskOrientedConversationPipeline'] @PIPELINES.register_module( - Tasks.dialog_modeling, module_name=Pipelines.dialog_modeling) -class DialogModelingPipeline(Pipeline): + Tasks.task_oriented_conversation, + module_name=Pipelines.task_oriented_conversation) +class TaskOrientedConversationPipeline(Pipeline): def __init__(self, model: Union[SpaceForDialogModeling, str], diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 09d26c1e..30cbe923 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -79,7 +79,7 @@ class NLPTasks(object): token_classification = 'token-classification' conversational = 'conversational' text_generation = 'text-generation' - dialog_modeling = 'dialog-modeling' + task_oriented_conversation = 'task-oriented-conversation' dialog_intent_prediction = 'dialog-intent-prediction' dialog_state_tracking = 'dialog-state-tracking' table_question_answering = 'table-question-answering' diff --git a/tests/pipelines/test_dialog_modeling.py b/tests/pipelines/test_task_oriented_conversation.py similarity index 79% rename from tests/pipelines/test_dialog_modeling.py rename to tests/pipelines/test_task_oriented_conversation.py index 0ce81dcd..ab24df88 100644 --- a/tests/pipelines/test_dialog_modeling.py +++ b/tests/pipelines/test_task_oriented_conversation.py @@ -6,13 +6,13 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.models import Model from modelscope.models.nlp import SpaceForDialogModeling from modelscope.pipelines import pipeline -from modelscope.pipelines.nlp import DialogModelingPipeline +from modelscope.pipelines.nlp import TaskOrientedConversationPipeline from modelscope.preprocessors import DialogModelingPreprocessor from modelscope.utils.constant import Tasks from modelscope.utils.test_utils import test_level -class DialogModelingTest(unittest.TestCase): +class TaskOrientedConversationTest(unittest.TestCase): model_id = 'damo/nlp_space_dialog-modeling' test_case = { 'sng0073': { @@ -92,7 +92,7 @@ class DialogModelingTest(unittest.TestCase): } def generate_and_print_dialog_response( - self, pipelines: List[DialogModelingPipeline]): + self, pipelines: List[TaskOrientedConversationPipeline]): result = {} for step, item in enumerate(self.test_case['sng0073']['log']): @@ -108,39 +108,37 @@ class DialogModelingTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_by_direct_model_download(self): - cache_path = snapshot_download(self.model_id) + cache_path = snapshot_download( + self.model_id, revision='task_oriented_conversation') preprocessor = DialogModelingPreprocessor(model_dir=cache_path) model = SpaceForDialogModeling( model_dir=cache_path, text_field=preprocessor.text_field, - config=preprocessor.config, - device='cpu') + config=preprocessor.config) pipelines = [ - DialogModelingPipeline( - model=model, preprocessor=preprocessor, device='cpu'), + TaskOrientedConversationPipeline( + model=model, preprocessor=preprocessor), pipeline( - task=Tasks.dialog_modeling, + task=Tasks.task_oriented_conversation, model=model, - preprocessor=preprocessor, - device='cpu') + preprocessor=preprocessor) ] self.generate_and_print_dialog_response(pipelines) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_from_modelhub(self): - model = Model.from_pretrained(self.model_id) - preprocessor = DialogModelingPreprocessor( - model_dir=model.model_dir, device='cpu') + model = Model.from_pretrained( + self.model_id, revision='task_oriented_conversation') + preprocessor = DialogModelingPreprocessor(model_dir=model.model_dir) pipelines = [ - DialogModelingPipeline( - model=model, preprocessor=preprocessor, device='cpu'), + TaskOrientedConversationPipeline( + model=model, preprocessor=preprocessor), pipeline( - task=Tasks.dialog_modeling, + task=Tasks.task_oriented_conversation, model=model, - preprocessor=preprocessor, - device='cpu') + preprocessor=preprocessor) ] self.generate_and_print_dialog_response(pipelines) @@ -149,17 +147,25 @@ class DialogModelingTest(unittest.TestCase): def test_run_with_model_name(self): pipelines = [ pipeline( - task=Tasks.dialog_modeling, model=self.model_id, device='cpu'), + task=Tasks.task_oriented_conversation, + model=self.model_id, + model_revision='task_oriented_conversation'), pipeline( - task=Tasks.dialog_modeling, model=self.model_id, device='cpu') + task=Tasks.task_oriented_conversation, + model=self.model_id, + model_revision='task_oriented_conversation') ] self.generate_and_print_dialog_response(pipelines) @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_with_default_model(self): pipelines = [ - pipeline(task=Tasks.dialog_modeling, device='cpu'), - pipeline(task=Tasks.dialog_modeling, device='cpu') + pipeline( + task=Tasks.task_oriented_conversation, + model_revision='task_oriented_conversation'), + pipeline( + task=Tasks.task_oriented_conversation, + model_revision='task_oriented_conversation') ] self.generate_and_print_dialog_response(pipelines)