diff --git a/data/test/images/keypoints_detect/test_img_face_2d_keypoints.png b/data/test/images/keypoints_detect/test_img_face_2d_keypoints.png new file mode 100644 index 00000000..00311c33 --- /dev/null +++ b/data/test/images/keypoints_detect/test_img_face_2d_keypoints.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:331ead75033fa2f01f6be72a2f8e34d581fcb593308067815d4bb136bb13b766 +size 54390 diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 3225710a..06b5a476 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -24,6 +24,7 @@ class Models(object): body_2d_keypoints = 'body-2d-keypoints' body_3d_keypoints = 'body-3d-keypoints' crowd_counting = 'HRNetCrowdCounting' + face_2d_keypoints = 'face-2d-keypoints' panoptic_segmentation = 'swinL-panoptic-segmentation' image_reid_person = 'passvitb' video_summarization = 'pgl-video-summarization' @@ -112,6 +113,7 @@ class Pipelines(object): object_detection = 'vit-object-detection' easycv_detection = 'easycv-detection' easycv_segmentation = 'easycv-segmentation' + face_2d_keypoints = 'mobilenet_face-2d-keypoints_alignment' salient_detection = 'u2net-salient-detection' image_classification = 'image-classification' face_detection = 'resnet-face-detection-scrfd10gkps' @@ -353,6 +355,7 @@ class Datasets(object): """ Names for different datasets. """ ClsDataset = 'ClsDataset' + Face2dKeypointsDataset = 'Face2dKeypointsDataset' SegDataset = 'SegDataset' DetDataset = 'DetDataset' DetImagesMixDataset = 'DetImagesMixDataset' diff --git a/modelscope/models/cv/__init__.py b/modelscope/models/cv/__init__.py index 331f23bd..4db43d17 100644 --- a/modelscope/models/cv/__init__.py +++ b/modelscope/models/cv/__init__.py @@ -3,9 +3,9 @@ # yapf: disable from . import (action_recognition, animal_recognition, body_2d_keypoints, body_3d_keypoints, cartoon, cmdssl_video_embedding, - crowd_counting, face_detection, face_generation, - image_classification, image_color_enhance, image_colorization, - image_denoise, image_instance_segmentation, + crowd_counting, face_2d_keypoints, face_detection, + face_generation, image_classification, image_color_enhance, + image_colorization, image_denoise, image_instance_segmentation, image_panoptic_segmentation, image_portrait_enhancement, image_reid_person, image_semantic_segmentation, image_to_image_generation, image_to_image_translation, diff --git a/modelscope/models/cv/face_2d_keypoints/__init__.py b/modelscope/models/cv/face_2d_keypoints/__init__.py new file mode 100644 index 00000000..636ba0f4 --- /dev/null +++ b/modelscope/models/cv/face_2d_keypoints/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .face_2d_keypoints_align import Face2DKeypoints + +else: + _import_structure = {'face_2d_keypoints_align': ['Face2DKeypoints']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py b/modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py new file mode 100644 index 00000000..468662a0 --- /dev/null +++ b/modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py @@ -0,0 +1,16 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from easycv.models.face.face_keypoint import FaceKeypoint + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.models.cv.easycv_base import EasyCVBaseModel +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + group_key=Tasks.face_2d_keypoints, module_name=Models.face_2d_keypoints) +class Face2DKeypoints(EasyCVBaseModel, FaceKeypoint): + + def __init__(self, model_dir=None, *args, **kwargs): + EasyCVBaseModel.__init__(self, model_dir, args, kwargs) + FaceKeypoint.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/cv/face_2d_keypoins/__init__.py b/modelscope/msdatasets/cv/face_2d_keypoins/__init__.py new file mode 100644 index 00000000..e9d76b7e --- /dev/null +++ b/modelscope/msdatasets/cv/face_2d_keypoins/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .face_2d_keypoints_dataset import FaceKeypointDataset + +else: + _import_structure = {'face_2d_keypoints_dataset': ['FaceKeypointDataset']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/modelscope/msdatasets/cv/face_2d_keypoins/face_2d_keypoints_dataset.py b/modelscope/msdatasets/cv/face_2d_keypoins/face_2d_keypoints_dataset.py new file mode 100644 index 00000000..a902999d --- /dev/null +++ b/modelscope/msdatasets/cv/face_2d_keypoins/face_2d_keypoints_dataset.py @@ -0,0 +1,13 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from easycv.datasets.face import FaceKeypointDataset as _FaceKeypointDataset + +from modelscope.metainfo import Datasets +from modelscope.msdatasets.task_datasets.builder import TASK_DATASETS +from modelscope.utils.constant import Tasks + + +@TASK_DATASETS.register_module( + group_key=Tasks.face_2d_keypoints, + module_name=Datasets.Face2dKeypointsDataset) +class FaceKeypointDataset(_FaceKeypointDataset): + """EasyCV dataset for face 2d keypoints.""" diff --git a/modelscope/outputs.py b/modelscope/outputs.py index 6fada2b0..e84c8dcc 100644 --- a/modelscope/outputs.py +++ b/modelscope/outputs.py @@ -57,6 +57,15 @@ TASK_OUTPUTS = { # } Tasks.ocr_recognition: [OutputKeys.TEXT], + # face 2d keypoint result for single sample + # { + # "keypoints": [ + # [x1, y1]*106 + # ], + # "poses": [pitch, roll, yaw] + # } + Tasks.face_2d_keypoints: [OutputKeys.KEYPOINTS, OutputKeys.POSES], + # face detection result for single sample # { # "scores": [0.9, 0.1, 0.05, 0.05] diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 40c237c8..f43d152b 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -103,6 +103,8 @@ DEFAULT_MODEL_FOR_PIPELINE = { 'damo/cv_resnet_facedetection_scrfd10gkps'), Tasks.face_recognition: (Pipelines.face_recognition, 'damo/cv_ir101_facerecognition_cfglint'), + Tasks.face_2d_keypoints: (Pipelines.face_2d_keypoints, + 'damo/cv_mobilenet_face-2d-keypoints_alignment'), Tasks.video_multi_modal_embedding: (Pipelines.video_multi_modal_embedding, 'damo/multi_modal_clip_vtretrival_msrvtt_53'), diff --git a/modelscope/pipelines/cv/__init__.py b/modelscope/pipelines/cv/__init__.py index c8cb0c6a..9e7d80ee 100644 --- a/modelscope/pipelines/cv/__init__.py +++ b/modelscope/pipelines/cv/__init__.py @@ -43,7 +43,7 @@ if TYPE_CHECKING: from .tinynas_classification_pipeline import TinynasClassificationPipeline from .video_category_pipeline import VideoCategoryPipeline from .virtual_try_on_pipeline import VirtualTryonPipeline - from .easycv_pipelines import EasyCVDetectionPipeline, EasyCVSegmentationPipeline + from .easycv_pipelines import EasyCVDetectionPipeline, EasyCVSegmentationPipeline, Face2DKeypointsPipeline from .text_driven_segmentation_pipleline import TextDrivenSegmentationPipleline from .movie_scene_segmentation_pipeline import MovieSceneSegmentationPipeline @@ -96,8 +96,10 @@ else: 'tinynas_classification_pipeline': ['TinynasClassificationPipeline'], 'video_category_pipeline': ['VideoCategoryPipeline'], 'virtual_try_on_pipeline': ['VirtualTryonPipeline'], - 'easycv_pipeline': - ['EasyCVDetectionPipeline', 'EasyCVSegmentationPipeline'], + 'easycv_pipeline': [ + 'EasyCVDetectionPipeline', 'EasyCVSegmentationPipeline', + 'Face2DKeypointsPipeline' + ], 'text_driven_segmentation_pipeline': ['TextDrivenSegmentationPipeline'], 'movie_scene_segmentation_pipeline': diff --git a/modelscope/pipelines/cv/easycv_pipelines/__init__.py b/modelscope/pipelines/cv/easycv_pipelines/__init__.py index 0984ff43..4f149130 100644 --- a/modelscope/pipelines/cv/easycv_pipelines/__init__.py +++ b/modelscope/pipelines/cv/easycv_pipelines/__init__.py @@ -6,10 +6,12 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .detection_pipeline import EasyCVDetectionPipeline from .segmentation_pipeline import EasyCVSegmentationPipeline + from .face_2d_keypoints_pipeline import Face2DKeypointsPipeline else: _import_structure = { 'detection_pipeline': ['EasyCVDetectionPipeline'], - 'segmentation_pipeline': ['EasyCVSegmentationPipeline'] + 'segmentation_pipeline': ['EasyCVSegmentationPipeline'], + 'face_2d_keypoints_pipeline': ['Face2DKeypointsPipeline'] } import sys diff --git a/modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py b/modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py new file mode 100644 index 00000000..eb4d6c15 --- /dev/null +++ b/modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py @@ -0,0 +1,41 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any + +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.builder import PIPELINES +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from .base import EasyCVPipeline + + +@PIPELINES.register_module( + Tasks.face_2d_keypoints, module_name=Pipelines.face_2d_keypoints) +class Face2DKeypointsPipeline(EasyCVPipeline): + """Pipeline for face 2d keypoints detection.""" + + def __init__(self, + model: str, + model_file_pattern=ModelFile.TORCH_MODEL_FILE, + *args, + **kwargs): + """ + model (str): model id on modelscope hub or local model path. + model_file_pattern (str): model file pattern. + """ + + super(Face2DKeypointsPipeline, self).__init__( + model=model, + model_file_pattern=model_file_pattern, + *args, + **kwargs) + + def show_result(self, img, points, scale=2, save_path=None): + return self.predict_op.show_result(img, points, scale, save_path) + + def __call__(self, inputs) -> Any: + output = self.predict_op(inputs)[0][0] + points = output['point'] + poses = output['pose'] + + return {OutputKeys.KEYPOINTS: points, OutputKeys.POSES: poses} diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index ed1ec798..86808ea1 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -20,6 +20,7 @@ class CVTasks(object): animal_recognition = 'animal-recognition' face_detection = 'face-detection' face_recognition = 'face-recognition' + face_2d_keypoints = 'face-2d-keypoints' human_detection = 'human-detection' human_object_interaction = 'human-object-interaction' face_image_generation = 'face-image-generation' diff --git a/tests/pipelines/test_face_2d_keypoints.py b/tests/pipelines/test_face_2d_keypoints.py new file mode 100644 index 00000000..a5e347e8 --- /dev/null +++ b/tests/pipelines/test_face_2d_keypoints.py @@ -0,0 +1,36 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +import cv2 + +from modelscope.outputs import OutputKeys +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.test_utils import test_level + + +class EasyCVFace2DKeypointsPipelineTest(unittest.TestCase): + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_face_2d_keypoints(self): + img_path = 'data/test/images/keypoints_detect/test_img_face_2d_keypoints.png' + model_id = 'damo/cv_mobilenet_face-2d-keypoints_alignment' + + face_2d_keypoints_align = pipeline( + task=Tasks.face_2d_keypoints, model=model_id) + output = face_2d_keypoints_align(img_path) + + output_keypoints = output[OutputKeys.KEYPOINTS] + output_pose = output[OutputKeys.POSES] + + img = cv2.imread(img_path) + img = face_2d_keypoints_align.show_result( + img, output_keypoints, scale=2, save_path='face_keypoints.jpg') + + self.assertEqual(output_keypoints.shape[0], 106) + self.assertEqual(output_keypoints.shape[1], 2) + self.assertEqual(output_pose.shape[0], 3) + + +if __name__ == '__main__': + unittest.main()