[to #44657982] add unittest for demo and demotest utils

unittest for demo service
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10006180

modelscope/utils/constant.py

@@ -136,6 +136,22 @@ class MultiModalTasks(object):
     image_text_retrieval = 'image-text-retrieval'
 
 
+class TasksIODescriptions(object):
+    image_to_image = 'image_to_image',
+    images_to_image = 'images_to_image',
+    image_to_text = 'image_to_text',
+    seed_to_image = 'seed_to_image',
+    text_to_speech = 'text_to_speech',
+    text_to_text = 'text_to_text',
+    speech_to_text = 'speech_to_text',
+    speech_to_speech = 'speech_to_speech'
+    speeches_to_speech = 'speeches_to_speech',
+    visual_grounding = 'visual_grounding',
+    visual_question_answering = 'visual_question_answering',
+    visual_entailment = 'visual_entailment',
+    generative_multi_modal_embedding = 'generative_multi_modal_embedding'
+
+
 class Tasks(CVTasks, NLPTasks, AudioTasks, MultiModalTasks):
     """ Names for tasks supported by modelscope.
 

modelscope/utils/demo_utils.py

@@ -0,0 +1,243 @@
+import io
+
+import cv2
+import json
+import numpy as np
+
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks, TasksIODescriptions
+
+TASKS_INPUT_TEMPLATES = {
+    # vision tasks
+    Tasks.image_portrait_stylization: TasksIODescriptions.image_to_image,
+    Tasks.portrait_matting: TasksIODescriptions.image_to_image,
+    Tasks.skin_retouching: TasksIODescriptions.image_to_image,
+    Tasks.image_captioning: TasksIODescriptions.image_to_text,
+    Tasks.image_denoising: TasksIODescriptions.image_to_image,
+    Tasks.image_portrait_enhancement: TasksIODescriptions.image_to_image,
+    Tasks.image_super_resolution: TasksIODescriptions.image_to_image,
+    Tasks.image_colorization: TasksIODescriptions.image_to_image,
+    Tasks.image_color_enhancement: TasksIODescriptions.image_to_image,
+    Tasks.face_image_generation: TasksIODescriptions.seed_to_image,
+    Tasks.image_style_transfer: TasksIODescriptions.images_to_image,
+    Tasks.image_segmentation: TasksIODescriptions.image_to_text,
+    Tasks.image_object_detection: TasksIODescriptions.image_to_text,
+
+    # not tested
+    Tasks.image_classification: TasksIODescriptions.image_to_text,
+    Tasks.ocr_detection: TasksIODescriptions.image_to_text,
+    Tasks.ocr_recognition: TasksIODescriptions.image_to_text,
+    Tasks.body_2d_keypoints: TasksIODescriptions.image_to_text,
+
+    # nlp tasks
+    Tasks.text_classification: TasksIODescriptions.text_to_text,
+    Tasks.text_generation: TasksIODescriptions.text_to_text,
+    Tasks.word_segmentation: TasksIODescriptions.text_to_text,
+    Tasks.text_error_correction: TasksIODescriptions.text_to_text,
+    Tasks.named_entity_recognition: TasksIODescriptions.text_to_text,
+    Tasks.sentiment_classification: TasksIODescriptions.text_to_text,
+
+    # audio tasks
+    Tasks.text_to_speech: TasksIODescriptions.text_to_speech,
+    Tasks.auto_speech_recognition: TasksIODescriptions.speech_to_text,
+    Tasks.keyword_spotting: TasksIODescriptions.speech_to_text,
+    Tasks.acoustic_noise_suppression: TasksIODescriptions.speech_to_speech,
+    Tasks.acoustic_echo_cancellation: TasksIODescriptions.speeches_to_speech,
+
+    # multi-modal
+    Tasks.visual_grounding: TasksIODescriptions.visual_grounding,
+    Tasks.visual_question_answering:
+    TasksIODescriptions.visual_question_answering,
+    Tasks.visual_entailment: TasksIODescriptions.visual_entailment,
+    Tasks.generative_multi_modal_embedding:
+    TasksIODescriptions.generative_multi_modal_embedding,
+
+    # new tasks
+    Tasks.virtual_try_on: TasksIODescriptions.images_to_image,
+
+    # TODO(lingcai.wl): support more tasks and implement corresponding example
+}
+
+INPUT_EXAMPLES = {
+    # Must align with task schema defined in the Widget section of model card=
+    # cv
+    TasksIODescriptions.image_to_image: {
+        'inputs': [
+            'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_cartoon.png'
+        ],
+        'urlPaths': {
+            'outUrls': [{
+                'outputKey': OutputKeys.OUTPUT_IMG,
+                'fileType': 'png'
+            }]
+        }
+    },
+    TasksIODescriptions.images_to_image: {
+        'inputs': [
+            'https://modelscope.oss-cn-beijing.aliyuncs.com/demo/image-style-transfer/style_transfer_content.jpg',
+            'https://modelscope.oss-cn-beijing.aliyuncs.com/demo/image-style-transfer/style_transfer_style.jpg'
+        ],
+        'urlPaths': {
+            'outUrls': [{
+                'outputKey': OutputKeys.OUTPUT_IMG,
+                'fileType': 'png'
+            }]
+        }
+    },
+    TasksIODescriptions.image_to_text: {
+        'inputs': [
+            'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_cartoon.png'
+        ],
+        'urlPaths': {}
+    },
+    # nlp
+    TasksIODescriptions.text_to_text: {
+        'inputs': ['test'],
+        'urlPaths': {}
+    },
+
+    # audio
+    TasksIODescriptions.speech_to_text: {
+        'inputs': [
+            'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example.wav'
+        ],
+        'urlPaths': {}
+    },
+    TasksIODescriptions.text_to_speech: {
+        'inputs': ['北京今天天气怎么样'],
+        'urlPaths': {
+            'outUrls': [{
+                'outputKey': OutputKeys.OUTPUT_PCM,
+                'fileType': 'pcm'
+            }]
+        }
+    },
+    TasksIODescriptions.speeches_to_speech: {
+        'inputs': [
+            'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/nearend_mic.wav',
+            'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/nearend_speech.wav'
+        ],
+        'urlPaths': {
+            'outUrls': [{
+                'outputKey': OutputKeys.OUTPUT_PCM,
+                'fileType': 'wav'
+            }]
+        }
+    },
+    TasksIODescriptions.speech_to_speech: {
+        'inputs': [
+            'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/speech_with_noise.wav'
+        ],
+        'urlPaths': {
+            'outUrls': [{
+                'outputKey': OutputKeys.OUTPUT_PCM,
+                'fileType': 'wav'
+            }]
+        }
+    },
+
+    # multi modal
+    TasksIODescriptions.visual_grounding: {
+        'task':
+        Tasks.visual_grounding,
+        'inputs': [
+            'http://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/visual-grounding/visual_grounding.png',
+            'a blue turtle-like pokemon with round head'
+        ],
+        'urlPaths': {}
+    },
+    TasksIODescriptions.visual_question_answering: {
+        'task':
+        Tasks.visual_question_answering,
+        'inputs': [
+            'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/visual_question_answering.png',
+            'what is grown on the plant?'
+        ],
+        'urlPaths': {}
+    },
+    TasksIODescriptions.visual_entailment: {
+        'task':
+        Tasks.visual_entailment,
+        'inputs': [
+            'http://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/visual-entailment/visual_entailment.jpg',
+            'there are two birds.', 'test'
+        ],
+        'urlPaths': {}
+    },
+    TasksIODescriptions.generative_multi_modal_embedding: {
+        'task':
+        Tasks.generative_multi_modal_embedding,
+        'inputs': [
+            'http://clip-multimodal.oss-cn-beijing.aliyuncs.com/lingchen/demo/dogs.jpg',
+            'dogs playing in the grass'
+        ],
+        'urlPaths': {}
+    },
+}
+
+
+class DemoCompatibilityCheck(object):
+
+    def compatibility_check(self):
+        if self.task not in TASKS_INPUT_TEMPLATES:
+            print('task is not supported in demo service so far')
+            return False
+        if TASKS_INPUT_TEMPLATES[self.task] not in INPUT_EXAMPLES:
+            print('no example input for this task')
+            return False
+
+        print('testing demo: ', self.task, self.model_id)
+        test_pipline = pipeline(self.task, self.model_id)
+        req = INPUT_EXAMPLES[TASKS_INPUT_TEMPLATES[self.task]]
+        output = test_pipline(preprocess(req))
+        json.dumps(output, cls=NumpyEncoder)
+        result = postprocess(req, output)
+        print(result)
+        return True
+
+
+class NumpyEncoder(json.JSONEncoder):
+
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+
+        if isinstance(obj, np.floating):
+            return float(obj)
+
+        if isinstance(obj, np.integer):
+            return int(obj)
+
+        return json.JSONEncoder.default(self, obj)
+
+
+def preprocess(req):
+    if len(req['inputs']) == 1:
+        inputs = req['inputs'][0]
+    else:
+        inputs = tuple(req['inputs'])
+    return inputs
+
+
+def postprocess(req, resp):
+    out_urls = req.get('urlPaths').get('outUrls')
+    if out_urls is None or len(out_urls) == 0:
+        return resp
+    new_resp = resp
+    if isinstance(resp, str):
+        new_resp = json.loads(resp)
+    for out_url in out_urls:
+        output_key = out_url['outputKey']
+        file_type = out_url['fileType']
+        new_resp.get(output_key)
+        if file_type == 'png' or file_type == 'jpg':
+            content = new_resp.get(output_key)
+            _, img_encode = cv2.imencode('.' + file_type, content)
+            img_bytes = img_encode.tobytes()
+            return type(img_bytes)
+        elif file_type == 'wav':
+            out_mem_file = io.BytesIO()
+            out_mem_file.write(new_resp.get(output_key))
+            return type(out_mem_file)
+        # TODO(lingcai.wl): support more file type

tests/pipelines/test_action_detection.py

@@ -2,21 +2,28 @@
 import unittest
 
 from modelscope.pipelines import pipeline
-from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ActionDetectionTest(unittest.TestCase):
+class ActionDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.action_detection
+        self.model_id = 'damo/cv_ResNetC3D_action-detection_detection2d'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run(self):
-        action_detection_pipline = pipeline(
-            Tasks.action_detection,
-            model='damo/cv_ResNetC3D_action-detection_detection2d')
+        action_detection_pipline = pipeline(self.task, model=self.model_id)
         result = action_detection_pipline(
             'data/test/videos/action_detection_test_video.mp4')
         print('action detection results:', result)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_action_recognition.py

@@ -1,24 +1,21 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-# !/usr/bin/env python
-import os.path as osp
-import tempfile
 import unittest
 
-from modelscope.fileio import File
 from modelscope.pipelines import pipeline
-from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ActionRecognitionTest(unittest.TestCase):
+class ActionRecognitionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.action_recognition
         self.model_id = 'damo/cv_TAdaConv_action-recognition'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
-        recognition_pipeline = pipeline(
-            Tasks.action_recognition, model=self.model_id)
+        recognition_pipeline = pipeline(self.task, self.model_id)
         result = recognition_pipeline(
             'data/test/videos/action_recognition_test_video.mp4')
 
@@ -26,12 +23,16 @@ class ActionRecognitionTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     def test_run_modelhub_default_model(self):
-        recognition_pipeline = pipeline(Tasks.action_recognition)
+        recognition_pipeline = pipeline(self.task)
         result = recognition_pipeline(
             'data/test/videos/action_recognition_test_video.mp4')
 
         print(f'recognition output: {result}.')
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_animal_recognition.py

@@ -2,19 +2,27 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class AnimalRecognitionTest(unittest.TestCase):
+class AnimalRecognitionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.animal_recognition
+        self.model_id = 'damo/cv_resnest101_animal_recognition'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run(self):
         animal_recognition = pipeline(
-            Tasks.animal_recognition,
-            model='damo/cv_resnest101_animal_recognition')
+            Tasks.animal_recognition, model=self.model_id)
         result = animal_recognition('data/test/images/dogs.jpg')
         print(result)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_automatic_speech_recognition.py

@@ -10,6 +10,7 @@ import soundfile
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import ColorCodes, Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.logger import get_logger
 from modelscope.utils.test_utils import download_and_untar, test_level
 
@@ -22,7 +23,8 @@ LITTLE_TESTSETS_FILE = 'data_aishell.tar.gz'
 LITTLE_TESTSETS_URL = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/datasets/data_aishell.tar.gz'
 
 
-class AutomaticSpeechRecognitionTest(unittest.TestCase):
+class AutomaticSpeechRecognitionTest(unittest.TestCase,
+                                     DemoCompatibilityCheck):
     action_info = {
         'test_run_with_wav_pytorch': {
             'checking_item': OutputKeys.TEXT,
@@ -74,6 +76,7 @@ class AutomaticSpeechRecognitionTest(unittest.TestCase):
         self.am_tf_model_id = 'damo/speech_paraformer_asr_nat-zh-cn-16k-common-vocab8358-tensorflow1'
         # this temporary workspace dir will store waveform files
         self.workspace = os.path.join(os.getcwd(), '.tmp')
+        self.task = Tasks.auto_speech_recognition
         if not os.path.exists(self.workspace):
             os.mkdir(self.workspace)
 
@@ -254,6 +257,10 @@ class AutomaticSpeechRecognitionTest(unittest.TestCase):
             model_id=self.am_tf_model_id, audio_in=dataset_path)
         self.check_result('test_run_with_wav_dataset_tf', rec_result)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_body_2d_keypoints.py

@@ -2,20 +2,20 @@
 import unittest
 
 import cv2
-import numpy as np
 from PIL import Image
 
-from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import draw_keypoints
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class Body2DKeypointsTest(unittest.TestCase):
+class Body2DKeypointsTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.body_2d_keypoints
         self.model_id = 'damo/cv_hrnetv2w32_body-2d-keypoints_image'
         self.test_image = 'data/test/images/keypoints_detect/000000438862.jpg'
 
@@ -26,16 +26,18 @@ class Body2DKeypointsTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_run_modelhub_with_image_file(self):
-        body_2d_keypoints = pipeline(
-            Tasks.body_2d_keypoints, model=self.model_id)
+        body_2d_keypoints = pipeline(self.task, model=self.model_id)
         self.pipeline_inference(body_2d_keypoints, self.test_image)
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub_with_image_input(self):
-        body_2d_keypoints = pipeline(
-            Tasks.body_2d_keypoints, model=self.model_id)
+        body_2d_keypoints = pipeline(self.task, model=self.model_id)
         self.pipeline_inference(body_2d_keypoints, Image.open(self.test_image))
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_body_3d_keypoints.py

@@ -1,23 +1,23 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import pdb
 import unittest
 
 import cv2
 import numpy as np
-from PIL import Image
 
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class Body3DKeypointsTest(unittest.TestCase):
+class Body3DKeypointsTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_canonical_body-3d-keypoints_video'
         self.test_video = 'data/test/videos/Walking.54138969.mp4'
+        self.task = Tasks.body_3d_keypoints
 
     def pipeline_inference(self, pipeline: Pipeline, pipeline_input):
         output = pipeline(pipeline_input)
@@ -44,6 +44,10 @@ class Body3DKeypointsTest(unittest.TestCase):
         body_3d_keypoints = pipeline(Tasks.body_3d_keypoints)
         self.pipeline_inference(body_3d_keypoints, self.test_video)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_cmdssl_video_embedding.py

@@ -4,20 +4,28 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class CMDSSLVideoEmbeddingTest(unittest.TestCase):
+class CMDSSLVideoEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.video_embedding
+        self.model_id = 'damo/cv_r2p1d_video_embedding'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
-        videossl_pipeline = pipeline(
-            Tasks.video_embedding, model='damo/cv_r2p1d_video_embedding')
+        videossl_pipeline = pipeline(task=self.task, model=self.model_id)
         result = videossl_pipeline(
             'data/test/videos/action_recognition_test_video.mp4')
 
         print(f'video embedding output: {result}.')
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_conversational_text_to_sql.py

@@ -1,6 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import unittest
-from typing import List
 
 from modelscope.hub.snapshot_download import snapshot_download
 from modelscope.models import Model
@@ -9,11 +8,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import ConversationalTextToSqlPipeline
 from modelscope.preprocessors import ConversationalTextToSqlPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.nlp.nlp_utils import text2sql_tracking_and_print_results
 from modelscope.utils.test_utils import test_level
 
 
-class ConversationalTextToSql(unittest.TestCase):
+class ConversationalTextToSql(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.conversational_text_to_sql
+        self.model_id = 'damo/nlp_star_conversational-text-to-sql'
+
     model_id = 'damo/nlp_star_conversational-text-to-sql'
     test_case = {
         'database_id':
@@ -39,10 +44,7 @@ class ConversationalTextToSql(unittest.TestCase):
         pipelines = [
             ConversationalTextToSqlPipeline(
                 model=model, preprocessor=preprocessor),
-            pipeline(
-                task=Tasks.conversational_text_to_sql,
-                model=model,
-                preprocessor=preprocessor)
+            pipeline(task=self.task, model=model, preprocessor=preprocessor)
         ]
         text2sql_tracking_and_print_results(self.test_case, pipelines)
 
@@ -55,26 +57,24 @@ class ConversationalTextToSql(unittest.TestCase):
         pipelines = [
             ConversationalTextToSqlPipeline(
                 model=model, preprocessor=preprocessor),
-            pipeline(
-                task=Tasks.conversational_text_to_sql,
-                model=model,
-                preprocessor=preprocessor)
+            pipeline(task=self.task, model=model, preprocessor=preprocessor)
         ]
         text2sql_tracking_and_print_results(self.test_case, pipelines)
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_with_model_name(self):
-        pipelines = [
-            pipeline(
-                task=Tasks.conversational_text_to_sql, model=self.model_id)
-        ]
+        pipelines = [pipeline(task=self.task, model=self.model_id)]
         text2sql_tracking_and_print_results(self.test_case, pipelines)
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     def test_run_with_default_model(self):
-        pipelines = [pipeline(task=Tasks.conversational_text_to_sql)]
+        pipelines = [pipeline(task=self.task)]
         text2sql_tracking_and_print_results(self.test_case, pipelines)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_crowd_counting.py

@@ -8,17 +8,19 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import numpy_to_cv2img
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.logger import get_logger
 from modelscope.utils.test_utils import test_level
 
 logger = get_logger()
 
 
-class CrowdCountingTest(unittest.TestCase):
+class CrowdCountingTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.input_location = 'data/test/images/crowd_counting.jpg'
         self.model_id = 'damo/cv_hrnet_crowd-counting_dcanet'
+        self.task = Tasks.crowd_counting
 
     def save_result(self, result):
         print('scores:', result[OutputKeys.SCORES])
@@ -28,7 +30,7 @@ class CrowdCountingTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_crowd_counting(self):
-        crowd_counting = pipeline(Tasks.crowd_counting, model=self.model_id)
+        crowd_counting = pipeline(task=self.task, model=self.model_id)
         result = crowd_counting(self.input_location)
         if result:
             self.save_result(result)
@@ -37,7 +39,7 @@ class CrowdCountingTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_crowd_counting_with_image(self):
-        crowd_counting = pipeline(Tasks.crowd_counting, model=self.model_id)
+        crowd_counting = pipeline(task=self.task, model=self.model_id)
         img = Image.open(self.input_location)
         result = crowd_counting(img)
         if result:
@@ -47,13 +49,17 @@ class CrowdCountingTest(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     def test_crowd_counting_with_default_task(self):
-        crowd_counting = pipeline(Tasks.crowd_counting)
+        crowd_counting = pipeline(self.task)
         result = crowd_counting(self.input_location)
         if result:
             self.save_result(result)
         else:
             raise ValueError('process error')
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_csanmt_translation.py

@@ -3,31 +3,39 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class TranslationTest(unittest.TestCase):
+class TranslationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.translation
+        self.model_id = 'damo/nlp_csanmt_translation_zh2en'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_with_model_name_for_zh2en(self):
-        model_id = 'damo/nlp_csanmt_translation_zh2en'
         inputs = '声明补充说，沃伦的同事都深感震惊，并且希望他能够投案自首。'
-        pipeline_ins = pipeline(task=Tasks.translation, model=model_id)
+        pipeline_ins = pipeline(self.task, model=self.model_id)
         print(pipeline_ins(input=inputs))
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_with_model_name_for_en2zh(self):
         model_id = 'damo/nlp_csanmt_translation_en2zh'
         inputs = 'Elon Musk, co-founder and chief executive officer of Tesla Motors.'
-        pipeline_ins = pipeline(task=Tasks.translation, model=model_id)
+        pipeline_ins = pipeline(self.task, model=model_id)
         print(pipeline_ins(input=inputs))
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     def test_run_with_default_model(self):
         inputs = '声明补充说，沃伦的同事都深感震惊，并且希望他能够投案自首。'
-        pipeline_ins = pipeline(task=Tasks.translation)
+        pipeline_ins = pipeline(self.task)
         print(pipeline_ins(input=inputs))
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_dialog_intent_prediction.py

@@ -8,11 +8,16 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import DialogIntentPredictionPipeline
 from modelscope.preprocessors import DialogIntentPredictionPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class DialogIntentPredictionTest(unittest.TestCase):
-    model_id = 'damo/nlp_space_dialog-intent-prediction'
+class DialogIntentPredictionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.task_oriented_conversation
+        self.model_id = 'damo/nlp_space_dialog-intent-prediction'
+
     test_case = [
         'How do I locate my card?',
         'I still have not received my new card, I ordered over a week ago.'
@@ -61,13 +66,15 @@ class DialogIntentPredictionTest(unittest.TestCase):
     def test_run_with_model_name(self):
         pipelines = [
             pipeline(
-                task=Tasks.task_oriented_conversation,
-                model=self.model_id,
-                model_revision='update')
+                task=self.task, model=self.model_id, model_revision='update')
         ]
         for my_pipeline, item in list(zip(pipelines, self.test_case)):
             print(my_pipeline(item))
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_dialog_modeling.py

@@ -10,11 +10,16 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import DialogModelingPipeline
 from modelscope.preprocessors import DialogModelingPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class DialogModelingTest(unittest.TestCase):
-    model_id = 'damo/nlp_space_dialog-modeling'
+class DialogModelingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.task_oriented_conversation
+        self.model_id = 'damo/nlp_space_dialog-modeling'
+
     test_case = {
         'sng0073': {
             'goal': {
@@ -139,7 +144,7 @@ class DialogModelingTest(unittest.TestCase):
     def test_run_with_model_name(self):
         pipelines = [
             pipeline(
-                task=Tasks.task_oriented_conversation,
+                task=self.task,
                 model=self.model_id,
                 model_revision='task_oriented_conversation')
         ]
@@ -149,11 +154,14 @@ class DialogModelingTest(unittest.TestCase):
     def test_run_with_default_model(self):
         pipelines = [
             pipeline(
-                task=Tasks.task_oriented_conversation,
-                model_revision='task_oriented_conversation')
+                task=self.task, model_revision='task_oriented_conversation')
         ]
         self.generate_and_print_dialog_response(pipelines)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_dialog_state_tracking.py

@@ -8,12 +8,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import DialogStateTrackingPipeline
 from modelscope.preprocessors import DialogStateTrackingPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.nlp.nlp_utils import tracking_and_print_dialog_states
 from modelscope.utils.test_utils import test_level
 
 
-class DialogStateTrackingTest(unittest.TestCase):
-    model_id = 'damo/nlp_space_dialog-state-tracking'
+class DialogStateTrackingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.task_oriented_conversation
+        self.model_id = 'damo/nlp_space_dialog-state-tracking'
+
     test_case = [{
         'User-1':
         'Hi, I\'m looking for a train that is going to cambridge and arriving there by 20:45, '
@@ -103,10 +108,7 @@ class DialogStateTrackingTest(unittest.TestCase):
         pipelines = [
             DialogStateTrackingPipeline(
                 model=model, preprocessor=preprocessor),
-            pipeline(
-                task=Tasks.task_oriented_conversation,
-                model=model,
-                preprocessor=preprocessor)
+            pipeline(task=self.task, model=model, preprocessor=preprocessor)
         ]
 
         tracking_and_print_dialog_states(self.test_case, pipelines)
@@ -115,12 +117,14 @@ class DialogStateTrackingTest(unittest.TestCase):
     def test_run_with_model_name(self):
         pipelines = [
             pipeline(
-                task=Tasks.task_oriented_conversation,
-                model=self.model_id,
-                model_revision='update')
+                task=self.task, model=self.model_id, model_revision='update')
         ]
         tracking_and_print_dialog_states(self.test_case, pipelines)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_document_segmentation.py

@@ -6,13 +6,18 @@ from typing import Any, Dict
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.logger import get_logger
 from modelscope.utils.test_utils import test_level
 
 logger = get_logger()
 
 
-class DocumentSegmentationTest(unittest.TestCase):
+class DocumentSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.document_segmentation
+        self.model_id = 'damo/nlp_bert_document-segmentation_chinese-base'
 
     model_id = 'damo/nlp_bert_document-segmentation_chinese-base'
     eng_model_id = 'damo/nlp_bert_document-segmentation_english-base'
@@ -21,10 +26,8 @@ class DocumentSegmentationTest(unittest.TestCase):
     eng_sentences = 'The Saint Alexander Nevsky Church was established in 1936 by Archbishop Vitaly (Maximenko) () on a tract of land donated by Yulia Martinovna Plavskaya.The initial chapel, dedicated to the memory of the great prince St. Alexander Nevsky (1220–1263), was blessed in May, 1936.The church building was subsequently expanded three times.In 1987, ground was cleared for the construction of the new church and on September 12, 1989, on the Feast Day of St. Alexander Nevsky, the cornerstone was laid and the relics of St. Herman of Alaska placed in the foundation.The imposing edifice, completed in 1997, is the work of Nikolaus Karsanov, architect and Protopresbyter Valery Lukianov, engineer.Funds were raised through donations.The Great blessing of the cathedral took place on October 18, 1997 with seven bishops, headed by Metropolitan Vitaly Ustinov, and 36 priests and deacons officiating, some 800 faithful attended the festivity.The old church was rededicated to Our Lady of Tikhvin.Metropolitan Hilarion (Kapral) announced, that cathedral will officially become the episcopal See of the Ruling Bishop of the Eastern American Diocese and the administrative center of the Diocese on September 12, 2014.At present the parish serves the spiritual needs of 300 members.The parochial school instructs over 90 boys and girls in religion, Russian language and history.The school meets every Saturday.The choir is directed by Andrew Burbelo.The sisterhood attends to the needs of the church and a church council acts in the administration of the community.The cathedral is decorated by frescoes in the Byzantine style.The iconography project was fulfilled by Father Andrew Erastov and his students from 1995 until 2001.'  # noqa *
 
     def run_pipeline(self, model_id: str, documents: str) -> Dict[str, Any]:
-        p = pipeline(task=Tasks.document_segmentation, model=model_id)
-
+        p = pipeline(task=self.task, model=model_id)
         result = p(documents=documents)
-
         return result
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
@@ -51,6 +54,10 @@ class DocumentSegmentationTest(unittest.TestCase):
         for document in documents_list:
             print(document)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_face_detection.py

@@ -3,19 +3,19 @@ import os.path as osp
 import unittest
 
 import cv2
-import numpy as np
 
 from modelscope.msdatasets import MsDataset
-from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import draw_face_detection_result
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class FaceDetectionTest(unittest.TestCase):
+class FaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.face_detection
         self.model_id = 'damo/cv_resnet_facedetection_scrfd10gkps'
 
     def show_result(self, img_path, detection_result):
@@ -49,6 +49,10 @@ class FaceDetectionTest(unittest.TestCase):
         result = face_detection(img_path)
         self.show_result(img_path, result)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_face_image_generation.py

@@ -8,12 +8,14 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class FaceGenerationTest(unittest.TestCase):
+class FaceGenerationTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.face_image_generation
         self.model_id = 'damo/cv_gan_face-image-generation'
 
     def pipeline_inference(self, pipeline: Pipeline, seed: int):
@@ -26,7 +28,7 @@ class FaceGenerationTest(unittest.TestCase):
     def test_run_modelhub(self):
         seed = 10
         face_generation = pipeline(
-            Tasks.face_image_generation,
+            self.task,
             model=self.model_id,
         )
         self.pipeline_inference(face_generation, seed)
@@ -34,9 +36,13 @@ class FaceGenerationTest(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     def test_run_modelhub_default_model(self):
         seed = 10
-        face_generation = pipeline(Tasks.face_image_generation)
+        face_generation = pipeline(self.task)
         self.pipeline_inference(face_generation, seed)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_face_recognition.py

@@ -6,12 +6,14 @@ import numpy as np
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class FaceRecognitionTest(unittest.TestCase):
+class FaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.face_recognition
         self.model_id = 'damo/cv_ir101_facerecognition_cfglint'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
@@ -26,6 +28,10 @@ class FaceRecognitionTest(unittest.TestCase):
         sim = np.dot(emb1[0], emb2[0])
         print(f'Cos similarity={sim:.3f}, img1:{img1}  img2:{img2}')
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_faq_question_answering.py

@@ -11,11 +11,16 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import FaqQuestionAnsweringPipeline
 from modelscope.preprocessors import FaqQuestionAnsweringPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class FaqQuestionAnsweringTest(unittest.TestCase):
-    model_id = 'damo/nlp_structbert_faq-question-answering_chinese-base'
+class FaqQuestionAnsweringTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.faq_question_answering
+        self.model_id = 'damo/nlp_structbert_faq-question-answering_chinese-base'
+
     param = {
         'query_set': ['如何使用优惠券', '在哪里领券', '在哪里领券'],
         'support_set': [{
@@ -80,6 +85,10 @@ class FaqQuestionAnsweringTest(unittest.TestCase):
             ['今天星期六', '明天星期几明天星期几'])
         print(np.shape(sentence_vec))
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_fill_mask.py

@@ -9,11 +9,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import FillMaskPipeline
 from modelscope.preprocessors import FillMaskPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
-class FillMaskTest(unittest.TestCase):
+class FillMaskTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.fill_mask
+        self.model_id = 'damo/nlp_veco_fill-mask-large'
+
     model_id_sbert = {
         'zh': 'damo/nlp_structbert_fill-mask_chinese-large',
         'en': 'damo/nlp_structbert_fill-mask_english-large'
@@ -134,6 +140,10 @@ class FillMaskTest(unittest.TestCase):
         print(f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: '
               f'{pipeline_ins(test_input)}\n')
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_general_image_classification.py

@@ -2,10 +2,16 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class GeneralImageClassificationTest(unittest.TestCase):
+class GeneralImageClassificationTest(unittest.TestCase,
+                                     DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_classification
+        self.model_id = 'damo/cv_vit-base_image-classification_Dailylife-labels'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_ImageNet(self):
@@ -29,6 +35,10 @@ class GeneralImageClassificationTest(unittest.TestCase):
         result = general_image_classification('data/test/images/bird.JPEG')
         print(result)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_general_recognition.py

@@ -2,10 +2,15 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class GeneralRecognitionTest(unittest.TestCase):
+class GeneralRecognitionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.general_recognition
+        self.model_id = 'damo/cv_resnest101_general_recognition'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run(self):
@@ -15,6 +20,10 @@ class GeneralRecognitionTest(unittest.TestCase):
         result = general_recognition('data/test/images/dogs.jpg')
         print(result)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_generative_multi_modal_embedding.py

@@ -5,11 +5,16 @@ import unittest
 from modelscope.models import Model
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class GEMMMultiModalEmbeddingTest(unittest.TestCase):
-    model_id = 'damo/multi-modal_gemm-vit-large-patch14_generative-multi-modal-embedding'
+class GEMMMultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.generative_multi_modal_embedding
+        self.model_id = 'damo/multi-modal_gemm-vit-large-patch14_generative-multi-modal-embedding'
+
     test_input = {
         'image': 'data/test/images/generative_multimodal.jpg',
         'text':
@@ -63,6 +68,10 @@ class GEMMMultiModalEmbeddingTest(unittest.TestCase):
         output = generative_multi_modal_embedding_pipeline(test_input)
         print(output)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_hicossl_video_embedding.py

@@ -4,12 +4,14 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class HICOSSLVideoEmbeddingTest(unittest.TestCase):
+class HICOSSLVideoEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.video_embedding
         self.model_id = 'damo/cv_s3dg_video-embedding'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
@@ -21,6 +23,10 @@ class HICOSSLVideoEmbeddingTest(unittest.TestCase):
 
         print(f'video embedding output: {result}.')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_color_enhance.py

@@ -8,13 +8,15 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageColorEnhanceTest(unittest.TestCase):
+class ImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_csrnet_image-color-enhance-models'
+        self.task = Tasks.image_color_enhancement
 
     def pipeline_inference(self, pipeline: Pipeline, input_location: str):
         result = pipeline(input_location)
@@ -36,6 +38,10 @@ class ImageColorEnhanceTest(unittest.TestCase):
         self.pipeline_inference(img_color_enhance,
                                 'data/test/images/image_color_enhance.png')
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_colorization.py

@@ -8,14 +8,16 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageColorizationTest(unittest.TestCase):
+class ImageColorizationTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_unet_image-colorization'
         self.test_image = 'data/test/images/marilyn_monroe_4.jpg'
+        self.task = Tasks.image_colorization
 
     def pipeline_inference(self, pipeline: Pipeline, test_image: str):
         result = pipeline(test_image)
@@ -35,6 +37,10 @@ class ImageColorizationTest(unittest.TestCase):
         image_colorization = pipeline(Tasks.image_colorization)
         self.pipeline_inference(image_colorization, self.test_image)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_denoise.py

@@ -10,11 +10,16 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.cv import ImageDenoisePipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageDenoiseTest(unittest.TestCase):
-    model_id = 'damo/cv_nafnet_image-denoise_sidd'
+class ImageDenoiseTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_denoising
+        self.model_id = 'damo/cv_nafnet_image-denoise_sidd'
+
     demo_image_path = 'data/test/images/noisy-demo-1.png'
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
@@ -56,6 +61,10 @@ class ImageDenoiseTest(unittest.TestCase):
         w, h = denoise_img.size
         print('pipeline: the shape of output_img is {}x{}'.format(h, w))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_instance_segmentation.py

@@ -12,11 +12,16 @@ from modelscope.pipelines.cv import ImageInstanceSegmentationPipeline
 from modelscope.preprocessors import build_preprocessor
 from modelscope.utils.config import Config
 from modelscope.utils.constant import Fields, ModelFile, Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageInstanceSegmentationTest(unittest.TestCase):
-    model_id = 'damo/cv_swin-b_image-instance-segmentation_coco'
+class ImageInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_segmentation
+        self.model_id = 'damo/cv_swin-b_image-instance-segmentation_coco'
+
     image = 'data/test/images/image_instance_segmentation.jpg'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
@@ -56,6 +61,10 @@ class ImageInstanceSegmentationTest(unittest.TestCase):
         print(f'pipeline1:{pipeline1(input=self.image)[OutputKeys.LABELS]}')
         print(f'pipeline2: {pipeline2(input=self.image)[OutputKeys.LABELS]}')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_matting.py

@@ -1,19 +1,18 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import os.path as osp
-import tempfile
 import unittest
 
 import cv2
 
-from modelscope.fileio import File
 from modelscope.msdatasets import MsDataset
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageMattingTest(unittest.TestCase):
+class ImageMattingTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_unet_image-matting'
@@ -62,6 +61,10 @@ class ImageMattingTest(unittest.TestCase):
             f'Output written to dir: {osp.dirname(osp.abspath("result_0.png"))}'
         )
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_panoptic_segmentation.py

@@ -7,16 +7,20 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import panoptic_seg_masks_to_image
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImagePanopticSegmentationTest(unittest.TestCase):
+class ImagePanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_segmentation
+        self.model_id = 'damo/cv_swinL_panoptic-segmentation_cocopan'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_image_panoptic_segmentation(self):
         input_location = 'data/test/images/image_panoptic_segmentation.jpg'
-        model_id = 'damo/cv_swinL_panoptic-segmentation_cocopan'
-        pan_segmentor = pipeline(Tasks.image_segmentation, model=model_id)
+        pan_segmentor = pipeline(Tasks.image_segmentation, model=self.model_id)
         result = pan_segmentor(input_location)
 
         draw_img = panoptic_seg_masks_to_image(result[OutputKeys.MASKS])
@@ -26,8 +30,7 @@ class ImagePanopticSegmentationTest(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_image_panoptic_segmentation_from_PIL(self):
         input_location = 'data/test/images/image_panoptic_segmentation.jpg'
-        model_id = 'damo/cv_swinL_panoptic-segmentation_cocopan'
-        pan_segmentor = pipeline(Tasks.image_segmentation, model=model_id)
+        pan_segmentor = pipeline(Tasks.image_segmentation, model=self.model_id)
         PIL_array = PIL.Image.open(input_location)
         result = pan_segmentor(PIL_array)
 
@@ -35,6 +38,10 @@ class ImagePanopticSegmentationTest(unittest.TestCase):
         cv2.imwrite('result.jpg', draw_img)
         print('print test_image_panoptic_segmentation from PIL return success')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_portrait_enhancement.py

@@ -9,12 +9,14 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImagePortraitEnhancementTest(unittest.TestCase):
+class ImagePortraitEnhancementTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.image_portrait_enhancement
         self.model_id = 'damo/cv_gpen_image-portrait-enhancement'
         self.test_image = 'data/test/images/Solvay_conference_1927.png'
 
@@ -37,6 +39,10 @@ class ImagePortraitEnhancementTest(unittest.TestCase):
         face_enhancement = pipeline(Tasks.image_portrait_enhancement)
         self.pipeline_inference(face_enhancement, self.test_image)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_reid_person.py

@@ -6,14 +6,16 @@ from PIL import Image
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageReidPersonTest(unittest.TestCase):
+class ImageReidPersonTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.input_location = 'data/test/images/image_reid_person.jpg'
         self.model_id = 'damo/cv_passvitb_image-reid-person_market'
+        self.task = Tasks.image_reid_person
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_image_reid_person(self):
@@ -48,6 +50,10 @@ class ImageReidPersonTest(unittest.TestCase):
         )
         print(f'The img embedding is: {result[OutputKeys.IMG_EMBEDDING]}')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_semantic_segmentation.py

@@ -7,17 +7,20 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import semantic_seg_masks_to_image
-from modelscope.utils.logger import get_logger
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageSemanticSegmentationTest(unittest.TestCase):
+class ImageSemanticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = 'image-segmentation'
+        self.model_id = 'damo/cv_swinL_semantic-segmentation_cocopanmerge'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_image_semantic_segmentation_panmerge(self):
         input_location = 'data/test/images/image_semantic_segmentation.jpg'
-        model_id = 'damo/cv_swinL_semantic-segmentation_cocopanmerge'
-        segmenter = pipeline(Tasks.image_segmentation, model=model_id)
+        segmenter = pipeline(Tasks.image_segmentation, model=self.model_id)
         result = segmenter(input_location)
 
         draw_img = semantic_seg_masks_to_image(result[OutputKeys.MASKS])
@@ -34,8 +37,7 @@ class ImageSemanticSegmentationTest(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_image_semantic_segmentation_vitadapter(self):
         input_location = 'data/test/images/image_semantic_segmentation.jpg'
-        model_id = 'damo/cv_vitadapter_semantic-segmentation_cocostuff164k'
-        segmenter = pipeline(Tasks.image_segmentation, model=model_id)
+        segmenter = pipeline(Tasks.image_segmentation, model=self.model_id)
         result = segmenter(input_location)
 
         draw_img = semantic_seg_masks_to_image(result[OutputKeys.MASKS])
@@ -49,6 +51,10 @@ class ImageSemanticSegmentationTest(unittest.TestCase):
         cv2.imwrite('result.jpg', draw_img)
         print('test_image_semantic_segmentation_vitadapter_from_PIL DONE')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_style_transfer.py

@@ -7,12 +7,14 @@ from modelscope.hub.snapshot_download import snapshot_download
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageStyleTransferTest(unittest.TestCase):
+class ImageStyleTransferTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.image_style_transfer
         self.model_id = 'damo/cv_aams_style-transfer_damo'
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
@@ -48,6 +50,10 @@ class ImageStyleTransferTest(unittest.TestCase):
         cv2.imwrite('result_styletransfer3.png', result[OutputKeys.OUTPUT_IMG])
         print('style_transfer.test_run_modelhub_default_model done')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_image_super_resolution.py

@@ -8,14 +8,16 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageSuperResolutionTest(unittest.TestCase):
+class ImageSuperResolutionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_rrdb_image-super-resolution'
         self.img = 'data/test/images/dogs.jpg'
+        self.task = Tasks.image_super_resolution
 
     def pipeline_inference(self, pipeline: Pipeline, img: str):
         result = pipeline(img)
@@ -35,6 +37,10 @@ class ImageSuperResolutionTest(unittest.TestCase):
         super_resolution = pipeline(Tasks.image_super_resolution)
         self.pipeline_inference(super_resolution, self.img)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_key_word_spotting.py

@@ -10,6 +10,7 @@ import soundfile
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import ColorCodes, Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.logger import get_logger
 from modelscope.utils.test_utils import download_and_untar, test_level
 
@@ -25,7 +26,7 @@ NEG_TESTSETS_FILE = 'neg_testsets.tar.gz'
 NEG_TESTSETS_URL = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/neg_testsets.tar.gz'
 
 
-class KeyWordSpottingTest(unittest.TestCase):
+class KeyWordSpottingTest(unittest.TestCase, DemoCompatibilityCheck):
     action_info = {
         'test_run_with_wav': {
             'checking_item': [OutputKeys.KWS_LIST, 0, 'keyword'],
@@ -272,6 +273,10 @@ class KeyWordSpottingTest(unittest.TestCase):
             model_id=self.model_id, audio_in=audio_list)
         self.check_result('test_run_with_roc', kws_result)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_live_category.py

@@ -3,20 +3,28 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class LiveCategoryTest(unittest.TestCase):
+class LiveCategoryTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.live_category
+        self.model_id = 'damo/cv_resnet50_live-category'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
-        category_pipeline = pipeline(
-            Tasks.live_category, model='damo/cv_resnet50_live-category')
+        category_pipeline = pipeline(Tasks.live_category, self.model_id)
         result = category_pipeline(
             'data/test/videos/live_category_test_video.mp4')
 
         print(f'live category output: {result}.')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_movie_scene_segmentation.py

@@ -3,17 +3,21 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class MovieSceneSegmentationTest(unittest.TestCase):
+class MovieSceneSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.movie_scene_segmentation
+        self.model_id = 'damo/cv_resnet50-bert_video-scene-segmentation_movienet'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_movie_scene_segmentation(self):
         input_location = 'data/test/videos/movie_scene_segmentation_test_video.mp4'
-        model_id = 'damo/cv_resnet50-bert_video-scene-segmentation_movienet'
         movie_scene_segmentation_pipeline = pipeline(
-            Tasks.movie_scene_segmentation, model=model_id)
+            Tasks.movie_scene_segmentation, model=self.model_id)
         result = movie_scene_segmentation_pipeline(input_location)
         if result:
             print(result)
@@ -31,6 +35,10 @@ class MovieSceneSegmentationTest(unittest.TestCase):
         else:
             raise ValueError('process error')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_mplug_tasks.py

@@ -7,10 +7,15 @@ from modelscope.models import Model
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class MplugTasksTest(unittest.TestCase):
+class MplugTasksTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = 'visual-question-answering'
+        self.model_id = 'damo/mplug_visual-question-answering_coco_large_en'
 
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_run_with_image_captioning_with_model(self):
@@ -75,6 +80,10 @@ class MplugTasksTest(unittest.TestCase):
         result = pipeline_retrieval(input)
         print(result)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_multi_modal_embedding.py

@@ -8,11 +8,16 @@ from modelscope.models import Model
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class MultiModalEmbeddingTest(unittest.TestCase):
-    model_id = 'damo/multi-modal_clip-vit-base-patch16_zh'
+class MultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.multi_modal_embedding
+        self.model_id = 'damo/multi-modal_clip-vit-base-patch16_zh'
+
     test_input = {'text': '皮卡丘'}
     model_version = 'dev'
 
@@ -54,6 +59,10 @@ class MultiModalEmbeddingTest(unittest.TestCase):
         print('l2-norm: {}'.format(torch.norm(text_embedding,
                                               dim=-1).item()))  # should be 1.0
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_named_entity_recognition.py

@@ -9,10 +9,16 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import NamedEntityRecognitionPipeline
 from modelscope.preprocessors import NERPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class NamedEntityRecognitionTest(unittest.TestCase):
+class NamedEntityRecognitionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.named_entity_recognition
+        self.model_id = 'damo/nlp_raner_named-entity-recognition_chinese-base-news'
+
     tcrf_model_id = 'damo/nlp_raner_named-entity-recognition_chinese-base-news'
     lcrf_model_id = 'damo/nlp_lstm_named-entity-recognition_chinese-news'
     sentence = '这与温岭市新河镇的一个神秘的传说有关。'
@@ -88,6 +94,10 @@ class NamedEntityRecognitionTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.named_entity_recognition)
         print(pipeline_ins(input=self.sentence))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_nli.py

@@ -8,12 +8,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import PairSentenceClassificationPipeline
 from modelscope.preprocessors import PairSentenceClassificationPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
-class NLITest(unittest.TestCase):
-    model_id = 'damo/nlp_structbert_nli_chinese-base'
+class NLITest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.nli
+        self.model_id = 'damo/nlp_structbert_nli_chinese-base'
+
     sentence1 = '四川商务职业学院和四川财经职业学院哪个好？'
     sentence2 = '四川商务职业学院商务管理在哪个校区？'
     regress_tool = MsRegressTool(baseline=False)
@@ -52,6 +57,10 @@ class NLITest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.nli)
         print(pipeline_ins(input=(self.sentence1, self.sentence2)))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_object_detection.py

@@ -3,10 +3,15 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ObjectDetectionTest(unittest.TestCase):
+class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.human_detection
+        self.model_id = 'damo/cv_resnet18_human-detection'
 
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_object_detection(self):
@@ -50,6 +55,10 @@ class ObjectDetectionTest(unittest.TestCase):
         else:
             raise ValueError('process error')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_ocr_detection.py

@@ -4,14 +4,16 @@ import unittest
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class OCRDetectionTest(unittest.TestCase):
+class OCRDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_resnet18_ocr-detection-line-level_damo'
         self.test_image = 'data/test/images/ocr_detection.jpg'
+        self.task = Tasks.ocr_detection
 
     def pipeline_inference(self, pipeline: Pipeline, input_location: str):
         result = pipeline(input_location)
@@ -28,6 +30,10 @@ class OCRDetectionTest(unittest.TestCase):
         ocr_detection = pipeline(Tasks.ocr_detection)
         self.pipeline_inference(ocr_detection, self.test_image)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_ocr_recognition.py

@@ -1,26 +1,21 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path as osp
-import shutil
-import sys
-import tempfile
 import unittest
-from typing import Any, Dict, List, Tuple, Union
 
-import cv2
-import numpy as np
 import PIL
 
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class OCRRecognitionTest(unittest.TestCase):
+class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_convnextTiny_ocr-recognition-general_damo'
         self.test_image = 'data/test/images/ocr_recognition.jpg'
+        self.task = Tasks.ocr_recognition
 
     def pipeline_inference(self, pipeline: Pipeline, input_location: str):
         result = pipeline(input_location)
@@ -42,6 +37,10 @@ class OCRRecognitionTest(unittest.TestCase):
         ocr_recognition = pipeline(Tasks.ocr_recognition)
         self.pipeline_inference(ocr_recognition, self.test_image)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_ofa_tasks.py

@@ -11,10 +11,11 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import created_boxed_image
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class OfaTasksTest(unittest.TestCase):
+class OfaTasksTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.output_dir = 'unittest_output'
@@ -251,6 +252,10 @@ class OfaTasksTest(unittest.TestCase):
         result[OutputKeys.OUTPUT_IMG].save('result.png')
         print(f'Output written to {osp.abspath("result.png")}')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_person_image_cartoon.py

@@ -8,13 +8,15 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ImageCartoonTest(unittest.TestCase):
+class ImageCartoonTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_unet_person-image-cartoon_compound-models'
+        self.task = Tasks.image_portrait_stylization
         self.test_image = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_cartoon.png'
 
     def pipeline_inference(self, pipeline: Pipeline, input_location: str):
@@ -34,6 +36,10 @@ class ImageCartoonTest(unittest.TestCase):
         img_cartoon = pipeline(Tasks.image_portrait_stylization)
         self.pipeline_inference(img_cartoon, self.test_image)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_product_retrieval_embedding.py

@@ -6,11 +6,16 @@ from modelscope.models import Model
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class ProductRetrievalEmbeddingTest(unittest.TestCase):
-    model_id = 'damo/cv_resnet50_product-bag-embedding-models'
+class ProductRetrievalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.product_retrieval_embedding
+        self.model_id = 'damo/cv_resnet50_product-bag-embedding-models'
+
     img_input = 'data/test/images/product_embed_bag.jpg'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
@@ -34,6 +39,10 @@ class ProductRetrievalEmbeddingTest(unittest.TestCase):
         result = product_embed(self.img_input)[OutputKeys.IMG_EMBEDDING]
         print('abs sum value is: {}'.format(np.sum(np.abs(result))))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_realtime_object_detection.py

@@ -2,22 +2,22 @@
 import unittest
 
 import cv2
-import numpy as np
 
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
-from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import realtime_object_detection_bbox_vis
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class RealtimeObjectDetectionTest(unittest.TestCase):
+class RealtimeObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/cv_cspnet_image-object-detection_yolox'
         self.model_nano_id = 'damo/cv_cspnet_image-object-detection_yolox_nano_coco'
         self.test_image = 'data/test/images/keypoints_detect/000000438862.jpg'
+        self.task = Tasks.image_object_detection
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
@@ -47,6 +47,10 @@ class RealtimeObjectDetectionTest(unittest.TestCase):
         else:
             raise ValueError('process error')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_relation_extraction.py

@@ -1,8 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import unittest
 
-import torch
-
 from modelscope.hub.snapshot_download import snapshot_download
 from modelscope.models import Model
 from modelscope.models.nlp import InformationExtractionModel
@@ -10,11 +8,16 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import InformationExtractionPipeline
 from modelscope.preprocessors import RelationExtractionPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class RelationExtractionTest(unittest.TestCase):
-    model_id = 'damo/nlp_bert_relation-extraction_chinese-base'
+class RelationExtractionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.information_extraction
+        self.model_id = 'damo/nlp_bert_relation-extraction_chinese-base'
+
     sentence = '高捷，祖籍江苏，本科毕业于东南大学'
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
@@ -52,6 +55,10 @@ class RelationExtractionTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.information_extraction)
         print(pipeline_ins(input=self.sentence))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_salient_detection.py

@@ -4,10 +4,15 @@ import unittest
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class SalientDetectionTest(unittest.TestCase):
+class SalientDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_segmentation
+        self.model_id = 'damo/cv_u2net_salient-detection'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_salient_detection(self):
@@ -19,6 +24,10 @@ class SalientDetectionTest(unittest.TestCase):
         # result[OutputKeys.MASKS] is salient map result,other keys are not used
         cv2.imwrite(input_location + '_salient.jpg', result[OutputKeys.MASKS])
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_sentence_similarity.py

@@ -8,12 +8,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import PairSentenceClassificationPipeline
 from modelscope.preprocessors import PairSentenceClassificationPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
-class SentenceSimilarityTest(unittest.TestCase):
-    model_id = 'damo/nlp_structbert_sentence-similarity_chinese-base'
+class SentenceSimilarityTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.sentence_similarity
+        self.model_id = 'damo/nlp_structbert_sentence-similarity_chinese-base'
+
     sentence1 = '今天气温比昨天高么？'
     sentence2 = '今天湿度比昨天高么？'
     regress_tool = MsRegressTool(baseline=False)
@@ -58,6 +63,10 @@ class SentenceSimilarityTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.sentence_similarity)
         print(pipeline_ins(input=(self.sentence1, self.sentence2)))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_sentiment_classification.py

@@ -9,11 +9,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import SingleSentenceClassificationPipeline
 from modelscope.preprocessors import SingleSentenceClassificationPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class SentimentClassificationTaskModelTest(unittest.TestCase):
-    model_id = 'damo/nlp_structbert_sentiment-classification_chinese-base'
+class SentimentClassificationTaskModelTest(unittest.TestCase,
+                                           DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.sentiment_classification
+        self.model_id = 'damo/nlp_structbert_sentiment-classification_chinese-base'
+
     sentence1 = '启动的时候很大声音，然后就会听到1.2秒的卡察的声音，类似齿轮摩擦的声音'
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
@@ -60,6 +66,10 @@ class SentimentClassificationTaskModelTest(unittest.TestCase):
         self.assertTrue(
             isinstance(pipeline_ins.model, SequenceClassificationModel))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_skin_retouching.py

@@ -9,12 +9,14 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.base import Pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class SkinRetouchingTest(unittest.TestCase):
+class SkinRetouchingTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.skin_retouching
         self.model_id = 'damo/cv_unet_skin-retouching'
         self.test_image = 'data/test/images/skin_retouching.png'
 
@@ -39,6 +41,10 @@ class SkinRetouchingTest(unittest.TestCase):
         skin_retouching = pipeline(Tasks.skin_retouching)
         self.pipeline_inference(skin_retouching, self.test_image)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_speech_signal_process.py

@@ -1,11 +1,10 @@
 import os.path
-import shutil
 import unittest
 
-from modelscope.fileio import File
 from modelscope.metainfo import Pipelines
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 NEAREND_MIC_FILE = 'data/test/audios/nearend_mic.wav'
@@ -14,7 +13,7 @@ FAREND_SPEECH_FILE = 'data/test/audios/farend_speech.wav'
 NOISE_SPEECH_FILE = 'data/test/audios/speech_with_noise.wav'
 
 
-class SpeechSignalProcessTest(unittest.TestCase):
+class SpeechSignalProcessTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         pass
@@ -85,6 +84,10 @@ class SpeechSignalProcessTest(unittest.TestCase):
             ans(data, output_path=output_path)
         print(f'Processed audio saved to {output_path}')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_text_classification.py

@@ -6,14 +6,16 @@ from modelscope.msdatasets import MsDataset
 from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import SequenceClassificationPipeline
 from modelscope.preprocessors import SequenceClassificationPreprocessor
-from modelscope.utils.constant import Hubs, Tasks
+from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class SequenceClassificationTest(unittest.TestCase):
+class SequenceClassificationTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.model_id = 'damo/bert-base-sst2'
+        self.task = Tasks.text_classification
 
     def predict(self, pipeline_ins: SequenceClassificationPipeline):
         from easynlp.appzoo import load_dataset
@@ -87,6 +89,10 @@ class SequenceClassificationTest(unittest.TestCase):
         result = text_classification(dataset)
         self.printDataset(result)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_text_driven_segmentation.py

@@ -23,6 +23,10 @@ class TextDrivenSegmentationTest(unittest.TestCase):
         # result[OutputKeys.MASKS] is segment map result,other keys are not used
         cv2.imwrite(input_location + '_lseg.jpg', result[OutputKeys.MASKS])
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.test_demo()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_text_error_correction.py

@@ -8,11 +8,16 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import TextErrorCorrectionPipeline
 from modelscope.preprocessors import TextErrorCorrectionPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class TextErrorCorrectionTest(unittest.TestCase):
-    model_id = 'damo/nlp_bart_text-error-correction_chinese'
+class TextErrorCorrectionTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.text_error_correction
+        self.model_id = 'damo/nlp_bart_text-error-correction_chinese'
+
     input = '随着中国经济突飞猛近，建造工业与日俱增'
 
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
@@ -50,6 +55,10 @@ class TextErrorCorrectionTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.text_error_correction)
         print(pipeline_ins(self.input))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_text_generation.py

@@ -8,10 +8,11 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import TextGenerationPipeline
 from modelscope.preprocessors import TextGenerationPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class TextGenerationTest(unittest.TestCase):
+class TextGenerationTest(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
         self.palm_model_id_zh = 'damo/nlp_palm2.0_text-generation_chinese-base'
@@ -128,6 +129,10 @@ class TextGenerationTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.text_generation)
         print(pipeline_ins(self.palm_input_zh))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_text_to_image_synthesis.py

@@ -8,11 +8,16 @@ from modelscope.models import Model
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class TextToImageSynthesisTest(unittest.TestCase):
-    model_id = 'damo/cv_diffusion_text-to-image-synthesis_tiny'
+class TextToImageSynthesisTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.text_to_image_synthesis
+        self.model_id = 'damo/cv_diffusion_text-to-image-synthesis_tiny'
+
     test_text = {
         'text': '宇航员',
         'generator_ddim_timesteps': 2,
@@ -46,6 +51,10 @@ class TextToImageSynthesisTest(unittest.TestCase):
             self.test_text)[OutputKeys.OUTPUT_IMG]
         print(np.sum(np.abs(img)))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_text_to_speech.py

@@ -10,6 +10,7 @@ from scipy.io.wavfile import write
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.logger import get_logger
 from modelscope.utils.test_utils import test_level
 
@@ -18,22 +19,29 @@ import tensorflow as tf  # isort:skip
 logger = get_logger()
 
 
-class TextToSpeechSambertHifigan16kPipelineTest(unittest.TestCase):
+class TextToSpeechSambertHifigan16kPipelineTest(unittest.TestCase,
+                                                DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.text_to_speech
+        self.model_id = 'damo/speech_sambert-hifigan_tts_zhitian_emo_zh-cn_16k'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_pipeline(self):
         text = '今天北京天气怎么样？'
-        model_id = 'damo/speech_sambert-hifigan_tts_zhitian_emo_zh-cn_16k'
         voice = 'zhitian_emo'
 
-        sambert_hifigan_tts = pipeline(
-            task=Tasks.text_to_speech, model=model_id)
+        sambert_hifigan_tts = pipeline(task=self.task, model=self.model_id)
         self.assertTrue(sambert_hifigan_tts is not None)
         output = sambert_hifigan_tts(input=text, voice=voice)
         self.assertIsNotNone(output[OutputKeys.OUTPUT_PCM])
         pcm = output[OutputKeys.OUTPUT_PCM]
         write('output.wav', 16000, pcm)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_tinynas_classification.py

@@ -2,10 +2,15 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class TinyNASClassificationTest(unittest.TestCase):
+class TinyNASClassificationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_classification
+        self.model_id = 'damo/cv_tinynas_classification'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run(self):
@@ -14,6 +19,10 @@ class TinyNASClassificationTest(unittest.TestCase):
         result = tinynas_classification('data/test/images/image_wolf.jpeg')
         print(result)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_tinynas_detection.py

@@ -15,6 +15,10 @@ class TinynasObjectDetectionTest(unittest.TestCase):
             'data/test/images/image_detection.jpg')
         print(result)
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.test_demo()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_video_category.py

@@ -3,20 +3,28 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class VideoCategoryTest(unittest.TestCase):
+class VideoCategoryTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.video_category
+        self.model_id = 'damo/cv_resnet50_video-category'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
-        category_pipeline = pipeline(
-            Tasks.video_category, model='damo/cv_resnet50_video-category')
+        category_pipeline = pipeline(Tasks.video_category, self.model_id)
         result = category_pipeline(
             'data/test/videos/video_category_test_video.mp4')
 
         print(f'video category output: {result}.')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_video_multi_modal_embedding.py

@@ -4,15 +4,19 @@ import unittest
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.logger import get_logger
 from modelscope.utils.test_utils import test_level
 
 logger = get_logger()
 
 
-class VideoMultiModalEmbeddingTest(unittest.TestCase):
+class VideoMultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.video_multi_modal_embedding
+        self.model_id = 'damo/multi_modal_clip_vtretrival_msrvtt_53'
 
-    model_id = 'damo/multi_modal_clip_vtretrival_msrvtt_53'
     video_path = 'data/test/videos/multi_modal_test_video_9770.mp4'
     caption = ('a person is connecting something to system', None, None)
     _input = {'video': video_path, 'text': caption}
@@ -37,6 +41,10 @@ class VideoMultiModalEmbeddingTest(unittest.TestCase):
         logger.info('video feature: {}'.format(
             output['video_embedding'][0][0][0]))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_video_single_object_tracking.py

@@ -5,12 +5,14 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import show_video_tracking_result
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class SingleObjectTracking(unittest.TestCase):
+class SingleObjectTracking(unittest.TestCase, DemoCompatibilityCheck):
 
     def setUp(self) -> None:
+        self.task = Tasks.video_single_object_tracking
         self.model_id = 'damo/cv_vitb_video-single-object-tracking_ostrack'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
@@ -33,6 +35,10 @@ class SingleObjectTracking(unittest.TestCase):
         result = video_single_object_tracking((video_path, init_bbox))
         print('result is : ', result[OutputKeys.BOXES])
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_video_summarization.py

@@ -4,17 +4,21 @@ import unittest
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import show_video_summarization_result
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class VideoSummarizationTest(unittest.TestCase):
+class VideoSummarizationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.video_summarization
+        self.model_id = 'damo/cv_googlenet_pgl-video-summarization'
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_modelhub(self):
-        model_id = 'damo/cv_googlenet_pgl-video-summarization'
         video_path = 'data/test/videos/video_category_test_video.mp4'
         summarization_pipeline = pipeline(
-            Tasks.video_summarization, model=model_id)
+            Tasks.video_summarization, model=self.model_id)
         result = summarization_pipeline(video_path)
 
         print(f'video summarization output: \n{result}.')
@@ -29,6 +33,10 @@ class VideoSummarizationTest(unittest.TestCase):
 
         print(f'video summarization output:\n {result}.')
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_virtual_try_on.py

@@ -6,11 +6,16 @@ from PIL import Image
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.test_utils import test_level
 
 
-class VirtualTryonTest(unittest.TestCase):
-    model_id = 'damo/cv_daflow_virtual-try-on_base'
+class VirtualTryonTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.virtual_try_on
+        self.model_id = 'damo/cv_daflow_virtual-try-on_base'
+
     masked_model = Image.open('data/test/images/virtual_tryon_model.jpg')
     pose = Image.open('data/test/images/virtual_tryon_pose.jpg')
     cloth = Image.open('data/test/images/virtual_tryon_cloth.jpg')
@@ -29,6 +34,10 @@ class VirtualTryonTest(unittest.TestCase):
         img = pipeline_virtual_tryon(self.input_imgs)[OutputKeys.OUTPUT_IMG]
         cv2.imwrite('demo.jpg', img[:, :, ::-1])
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_word_segmentation.py

@@ -1,5 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import shutil
 import unittest
 
 from modelscope.hub.snapshot_download import snapshot_download
@@ -9,12 +8,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import WordSegmentationPipeline
 from modelscope.preprocessors import TokenClassificationPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
-class WordSegmentationTest(unittest.TestCase):
-    model_id = 'damo/nlp_structbert_word-segmentation_chinese-base'
+class WordSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.word_segmentation
+        self.model_id = 'damo/nlp_structbert_word-segmentation_chinese-base'
+
     sentence = '今天天气不错，适合出去游玩'
     sentence_eng = 'I am a program.'
     regress_tool = MsRegressTool(baseline=False)
@@ -55,6 +59,10 @@ class WordSegmentationTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.word_segmentation)
         print(pipeline_ins(input=self.sentence))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()

tests/pipelines/test_zero_shot_classification.py

@@ -8,12 +8,17 @@ from modelscope.pipelines import pipeline
 from modelscope.pipelines.nlp import ZeroShotClassificationPipeline
 from modelscope.preprocessors import ZeroShotClassificationPreprocessor
 from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
 from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
-class ZeroShotClassificationTest(unittest.TestCase):
-    model_id = 'damo/nlp_structbert_zero-shot-classification_chinese-base'
+class ZeroShotClassificationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.zero_shot_classification
+        self.model_id = 'damo/nlp_structbert_zero-shot-classification_chinese-base'
+
     sentence = '全新突破 解放军运20版空中加油机曝光'
     labels = ['文化', '体育', '娱乐', '财经', '家居', '汽车', '教育', '科技', '军事']
     template = '这篇文章的标题是{}'
@@ -65,6 +70,10 @@ class ZeroShotClassificationTest(unittest.TestCase):
         pipeline_ins = pipeline(task=Tasks.zero_shot_classification)
         print(pipeline_ins(input=self.sentence, candidate_labels=self.labels))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
 
 if __name__ == '__main__':
     unittest.main()