Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9533526master
| @@ -6,8 +6,8 @@ from typing import Dict | |||||
| import cv2 | import cv2 | ||||
| import numpy as np | import numpy as np | ||||
| from ..metainfo import Metrics | |||||
| from ..utils.registry import default_group | |||||
| from modelscope.metainfo import Metrics | |||||
| from modelscope.utils.registry import default_group | |||||
| from .base import Metric | from .base import Metric | ||||
| from .builder import METRICS, MetricKeys | from .builder import METRICS, MetricKeys | ||||
| @@ -2,10 +2,12 @@ import torch | |||||
| import torch.nn as nn | import torch.nn as nn | ||||
| import torch.nn.functional as F | import torch.nn.functional as F | ||||
| from ..layers.activations import RectifiedLinear, Sigmoid | |||||
| from ..layers.affine_transform import AffineTransform | |||||
| from ..layers.deep_fsmn import DeepFsmn | |||||
| from ..layers.uni_deep_fsmn import Conv2d, UniDeepFsmn | |||||
| from modelscope.models.audio.aec.layers.activations import (RectifiedLinear, | |||||
| Sigmoid) | |||||
| from modelscope.models.audio.aec.layers.affine_transform import AffineTransform | |||||
| from modelscope.models.audio.aec.layers.deep_fsmn import DeepFsmn | |||||
| from modelscope.models.audio.aec.layers.uni_deep_fsmn import (Conv2d, | |||||
| UniDeepFsmn) | |||||
| class MaskNet(nn.Module): | class MaskNet(nn.Module): | ||||
| @@ -1,12 +1,10 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
| from abc import ABC, abstractmethod | from abc import ABC, abstractmethod | ||||
| from typing import Dict, List, Union | |||||
| from typing import Dict, Union | |||||
| import numpy as np | |||||
| from ...utils.config import ConfigDict | |||||
| from ...utils.logger import get_logger | |||||
| from .base_model import Model | |||||
| from modelscope.models.base.base_model import Model | |||||
| from modelscope.utils.config import ConfigDict | |||||
| from modelscope.utils.logger import get_logger | |||||
| logger = get_logger() | logger = get_logger() | ||||
| @@ -1,13 +1,10 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
| import os.path | |||||
| import re | |||||
| from typing import Dict, Optional, Union | |||||
| from typing import Dict | |||||
| import torch | import torch | ||||
| from torch import nn | |||||
| from ...utils.logger import get_logger | |||||
| from .base_head import Head | |||||
| from modelscope.models.base.base_head import Head | |||||
| from modelscope.utils.logger import get_logger | |||||
| logger = get_logger(__name__) | logger = get_logger(__name__) | ||||
| @@ -1,6 +1,6 @@ | |||||
| import numpy as np | import numpy as np | ||||
| from ..config import config as cfg | |||||
| from modelscope.models.cv.cartoon.facelib.config import config as cfg | |||||
| class GroupTrack(): | class GroupTrack(): | ||||
| @@ -12,8 +12,9 @@ import torch | |||||
| import torch.nn as nn | import torch.nn as nn | ||||
| from torch import Tensor | from torch import Tensor | ||||
| from ..generate import search | |||||
| from .ngram_repeat_block import NGramRepeatBlock | |||||
| from modelscope.models.multi_modal.ofa.generate import search | |||||
| from modelscope.models.multi_modal.ofa.generate.ngram_repeat_block import \ | |||||
| NGramRepeatBlock | |||||
| def _expand_mask(mask: torch.Tensor, | def _expand_mask(mask: torch.Tensor, | ||||
| @@ -1,5 +1,5 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
| from ...utils.error import TENSORFLOW_IMPORT_WARNING | |||||
| from modelscope.utils.error import TENSORFLOW_IMPORT_WARNING | |||||
| from .backbones import * # noqa F403 | from .backbones import * # noqa F403 | ||||
| from .bert_for_sequence_classification import * # noqa F403 | from .bert_for_sequence_classification import * # noqa F403 | ||||
| from .heads import * # noqa F403 | from .heads import * # noqa F403 | ||||
| @@ -5,9 +5,11 @@ import torch | |||||
| import torch.nn as nn | import torch.nn as nn | ||||
| import torch.nn.functional as F | import torch.nn.functional as F | ||||
| from ..modules.embedder import Embedder | |||||
| from ..modules.transformer_block import TransformerBlock | |||||
| from .model_base import SpaceModelBase | |||||
| from modelscope.models.nlp.backbones.space.model.model_base import \ | |||||
| SpaceModelBase | |||||
| from modelscope.models.nlp.backbones.space.modules.embedder import Embedder | |||||
| from modelscope.models.nlp.backbones.space.modules.transformer_block import \ | |||||
| TransformerBlock | |||||
| class UnifiedTransformer(SpaceModelBase): | class UnifiedTransformer(SpaceModelBase): | ||||
| @@ -26,7 +26,7 @@ class SpaceForDialogModeling(Model): | |||||
| """ | """ | ||||
| super().__init__(model_dir, *args, **kwargs) | super().__init__(model_dir, *args, **kwargs) | ||||
| from ...trainers.nlp.space.trainer.gen_trainer import MultiWOZTrainer | |||||
| from modelscope.trainers.nlp.space.trainer.gen_trainer import MultiWOZTrainer | |||||
| self.model_dir = model_dir | self.model_dir = model_dir | ||||
| self.config = kwargs.pop( | self.config = kwargs.pop( | ||||
| 'config', | 'config', | ||||
| @@ -1,9 +1,6 @@ | |||||
| from typing import Any, Dict, Optional, Union | from typing import Any, Dict, Optional, Union | ||||
| import cv2 | |||||
| import numpy as np | |||||
| import torch | import torch | ||||
| from PIL import Image | |||||
| from torchvision import transforms | from torchvision import transforms | ||||
| from modelscope.metainfo import Pipelines | from modelscope.metainfo import Pipelines | ||||
| @@ -11,13 +8,12 @@ from modelscope.models.base import Model | |||||
| from modelscope.models.cv.image_color_enhance.image_color_enhance import \ | from modelscope.models.cv.image_color_enhance.image_color_enhance import \ | ||||
| ImageColorEnhance | ImageColorEnhance | ||||
| from modelscope.outputs import OutputKeys | from modelscope.outputs import OutputKeys | ||||
| from modelscope.pipelines.base import Input | |||||
| from modelscope.pipelines.base import Input, Pipeline | |||||
| from modelscope.pipelines.builder import PIPELINES | |||||
| from modelscope.preprocessors import (ImageColorEnhanceFinetunePreprocessor, | from modelscope.preprocessors import (ImageColorEnhanceFinetunePreprocessor, | ||||
| LoadImage, load_image) | |||||
| from modelscope.utils.constant import ModelFile, Tasks | |||||
| LoadImage) | |||||
| from modelscope.utils.constant import Tasks | |||||
| from modelscope.utils.logger import get_logger | from modelscope.utils.logger import get_logger | ||||
| from ..base import Pipeline | |||||
| from ..builder import PIPELINES | |||||
| logger = get_logger() | logger = get_logger() | ||||
| @@ -1,21 +1,17 @@ | |||||
| from typing import Any, Dict, Optional, Union | from typing import Any, Dict, Optional, Union | ||||
| import cv2 | |||||
| import numpy as np | |||||
| import torch | import torch | ||||
| from PIL import Image | |||||
| from torchvision import transforms | from torchvision import transforms | ||||
| from modelscope.metainfo import Pipelines | from modelscope.metainfo import Pipelines | ||||
| from modelscope.models import Model | from modelscope.models import Model | ||||
| from modelscope.models.cv import NAFNetForImageDenoise | from modelscope.models.cv import NAFNetForImageDenoise | ||||
| from modelscope.outputs import OutputKeys | from modelscope.outputs import OutputKeys | ||||
| from modelscope.pipelines.base import Input | |||||
| from modelscope.pipelines.base import Input, Pipeline | |||||
| from modelscope.pipelines.builder import PIPELINES | |||||
| from modelscope.preprocessors import ImageDenoisePreprocessor, LoadImage | from modelscope.preprocessors import ImageDenoisePreprocessor, LoadImage | ||||
| from modelscope.utils.constant import Tasks | from modelscope.utils.constant import Tasks | ||||
| from modelscope.utils.logger import get_logger | from modelscope.utils.logger import get_logger | ||||
| from ..base import Pipeline | |||||
| from ..builder import PIPELINES | |||||
| logger = get_logger() | logger = get_logger() | ||||
| @@ -13,10 +13,10 @@ from modelscope.hub.snapshot_download import snapshot_download | |||||
| from modelscope.metainfo import Pipelines | from modelscope.metainfo import Pipelines | ||||
| from modelscope.models.cv.virual_tryon.sdafnet import SDAFNet_Tryon | from modelscope.models.cv.virual_tryon.sdafnet import SDAFNet_Tryon | ||||
| from modelscope.outputs import OutputKeys | from modelscope.outputs import OutputKeys | ||||
| from modelscope.pipelines.base import Pipeline | |||||
| from modelscope.pipelines.builder import PIPELINES | |||||
| from modelscope.preprocessors import load_image | from modelscope.preprocessors import load_image | ||||
| from modelscope.utils.constant import ModelFile, Tasks | from modelscope.utils.constant import ModelFile, Tasks | ||||
| from ..base import Pipeline | |||||
| from ..builder import PIPELINES | |||||
| @PIPELINES.register_module( | @PIPELINES.register_module( | ||||
| @@ -1,13 +1,10 @@ | |||||
| from typing import Any, Dict | from typing import Any, Dict | ||||
| import torch | |||||
| from modelscope.metainfo import Pipelines | from modelscope.metainfo import Pipelines | ||||
| from modelscope.pipelines.base import Input | |||||
| from modelscope.pipelines.base import Input, Pipeline | |||||
| from modelscope.pipelines.builder import PIPELINES | |||||
| from modelscope.utils.constant import Tasks | from modelscope.utils.constant import Tasks | ||||
| from modelscope.utils.logger import get_logger | from modelscope.utils.logger import get_logger | ||||
| from ..base import Model, Pipeline | |||||
| from ..builder import PIPELINES | |||||
| logger = get_logger() | logger = get_logger() | ||||
| @@ -273,7 +273,7 @@ class FillMaskPreprocessor(NLPPreprocessorBase): | |||||
| super().__init__(model_dir, *args, **kwargs) | super().__init__(model_dir, *args, **kwargs) | ||||
| def build_tokenizer(self, model_dir): | def build_tokenizer(self, model_dir): | ||||
| from ..utils.hub import get_model_type | |||||
| from modelscope.utils.hub import get_model_type | |||||
| model_type = get_model_type(model_dir) | model_type = get_model_type(model_dir) | ||||
| if model_type in ['sbert', 'structbert', 'bert']: | if model_type in ['sbert', 'structbert', 'bert']: | ||||
| from sofa import SbertTokenizer | from sofa import SbertTokenizer | ||||
| @@ -5,13 +5,14 @@ from typing import Any, Dict | |||||
| import json | import json | ||||
| from ...metainfo import Preprocessors | |||||
| from ...utils.config import Config | |||||
| from ...utils.constant import Fields, ModelFile | |||||
| from ...utils.type_assert import type_assert | |||||
| from ..base import Preprocessor | |||||
| from ..builder import PREPROCESSORS | |||||
| from .fields.intent_field import IntentBPETextField | |||||
| from modelscope.metainfo import Preprocessors | |||||
| from modelscope.preprocessors.base import Preprocessor | |||||
| from modelscope.preprocessors.builder import PREPROCESSORS | |||||
| from modelscope.preprocessors.space.fields.intent_field import \ | |||||
| IntentBPETextField | |||||
| from modelscope.utils.config import Config | |||||
| from modelscope.utils.constant import Fields, ModelFile | |||||
| from modelscope.utils.type_assert import type_assert | |||||
| __all__ = ['DialogIntentPredictionPreprocessor'] | __all__ = ['DialogIntentPredictionPreprocessor'] | ||||
| @@ -3,13 +3,14 @@ | |||||
| import os | import os | ||||
| from typing import Any, Dict | from typing import Any, Dict | ||||
| from ...metainfo import Preprocessors | |||||
| from ...utils.config import Config | |||||
| from ...utils.constant import Fields, ModelFile | |||||
| from ...utils.type_assert import type_assert | |||||
| from ..base import Preprocessor | |||||
| from ..builder import PREPROCESSORS | |||||
| from .fields.gen_field import MultiWOZBPETextField | |||||
| from modelscope.metainfo import Preprocessors | |||||
| from modelscope.preprocessors.base import Preprocessor | |||||
| from modelscope.preprocessors.builder import PREPROCESSORS | |||||
| from modelscope.preprocessors.space.fields.gen_field import \ | |||||
| MultiWOZBPETextField | |||||
| from modelscope.utils.config import Config | |||||
| from modelscope.utils.constant import Fields, ModelFile | |||||
| from modelscope.utils.type_assert import type_assert | |||||
| __all__ = ['DialogModelingPreprocessor'] | __all__ = ['DialogModelingPreprocessor'] | ||||
| @@ -1,13 +1,12 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
| import os | |||||
| from typing import Any, Dict | from typing import Any, Dict | ||||
| from modelscope.metainfo import Preprocessors | |||||
| from modelscope.preprocessors.base import Preprocessor | |||||
| from modelscope.preprocessors.builder import PREPROCESSORS | |||||
| from modelscope.utils.constant import Fields | from modelscope.utils.constant import Fields | ||||
| from modelscope.utils.type_assert import type_assert | from modelscope.utils.type_assert import type_assert | ||||
| from ...metainfo import Preprocessors | |||||
| from ..base import Preprocessor | |||||
| from ..builder import PREPROCESSORS | |||||
| from .dst_processors import convert_examples_to_features, multiwoz22Processor | from .dst_processors import convert_examples_to_features, multiwoz22Processor | ||||
| __all__ = ['DialogStateTrackingPreprocessor'] | __all__ = ['DialogStateTrackingPreprocessor'] | ||||
| @@ -7,11 +7,11 @@ from itertools import chain | |||||
| import numpy as np | import numpy as np | ||||
| from ....utils.logger import get_logger | |||||
| from ....utils.nlp.space import ontology, utils | |||||
| from ....utils.nlp.space.db_ops import MultiWozDB | |||||
| from ....utils.nlp.space.utils import list2np | |||||
| from ..tokenizer import Tokenizer | |||||
| from modelscope.preprocessors.space.tokenizer import Tokenizer | |||||
| from modelscope.utils.logger import get_logger | |||||
| from modelscope.utils.nlp.space import ontology, utils | |||||
| from modelscope.utils.nlp.space.db_ops import MultiWozDB | |||||
| from modelscope.utils.nlp.space.utils import list2np | |||||
| logger = get_logger() | logger = get_logger() | ||||
| @@ -13,10 +13,10 @@ import json | |||||
| import numpy as np | import numpy as np | ||||
| from tqdm import tqdm | from tqdm import tqdm | ||||
| from ....utils.nlp.space import ontology, utils | |||||
| from ....utils.nlp.space.scores import hierarchical_set_score | |||||
| from ....utils.nlp.space.utils import list2np | |||||
| from ..tokenizer import Tokenizer | |||||
| from modelscope.preprocessors.space.tokenizer import Tokenizer | |||||
| from modelscope.utils.nlp.space import ontology | |||||
| from modelscope.utils.nlp.space.scores import hierarchical_set_score | |||||
| from modelscope.utils.nlp.space.utils import list2np | |||||
| class BPETextField(object): | class BPETextField(object): | ||||
| @@ -7,8 +7,8 @@ import numpy as np | |||||
| import torch | import torch | ||||
| from modelscope.trainers.hooks.hook import Hook | from modelscope.trainers.hooks.hook import Hook | ||||
| from modelscope.trainers.hooks.priority import Priority | |||||
| from modelscope.utils.constant import ModeKeys | from modelscope.utils.constant import ModeKeys | ||||
| from ..priority import Priority | |||||
| class LoggerHook(Hook): | class LoggerHook(Hook): | ||||
| @@ -8,10 +8,10 @@ import json | |||||
| import torch | import torch | ||||
| from torch import distributed as dist | from torch import distributed as dist | ||||
| from modelscope.trainers.hooks.builder import HOOKS | |||||
| from modelscope.trainers.hooks.logger.base import LoggerHook | |||||
| from modelscope.utils.constant import LogKeys, ModeKeys | from modelscope.utils.constant import LogKeys, ModeKeys | ||||
| from modelscope.utils.torch_utils import get_dist_info | from modelscope.utils.torch_utils import get_dist_info | ||||
| from ..builder import HOOKS | |||||
| from .base import LoggerHook | |||||
| @HOOKS.register_module() | @HOOKS.register_module() | ||||
| @@ -3,9 +3,9 @@ from typing import Dict, Optional, Tuple, Union | |||||
| import numpy as np | import numpy as np | ||||
| from modelscope.trainers.base import BaseTrainer | |||||
| from modelscope.trainers.builder import TRAINERS | |||||
| from modelscope.utils.logger import get_logger | from modelscope.utils.logger import get_logger | ||||
| from ..base import BaseTrainer | |||||
| from ..builder import TRAINERS | |||||
| PATH = None | PATH = None | ||||
| logger = get_logger(PATH) | logger = get_logger(PATH) | ||||
| @@ -13,8 +13,9 @@ import torch | |||||
| from tqdm import tqdm | from tqdm import tqdm | ||||
| from transformers.optimization import AdamW, get_linear_schedule_with_warmup | from transformers.optimization import AdamW, get_linear_schedule_with_warmup | ||||
| from .....utils.nlp.space import ontology | |||||
| from ..metrics.metrics_tracker import MetricsTracker | |||||
| from modelscope.trainers.nlp.space.metrics.metrics_tracker import \ | |||||
| MetricsTracker | |||||
| from modelscope.utils.nlp.space import ontology | |||||
| def get_logger(log_path, name='default'): | def get_logger(log_path, name='default'): | ||||
| @@ -14,7 +14,8 @@ import torch | |||||
| from tqdm import tqdm | from tqdm import tqdm | ||||
| from transformers.optimization import AdamW, get_linear_schedule_with_warmup | from transformers.optimization import AdamW, get_linear_schedule_with_warmup | ||||
| from ..metrics.metrics_tracker import MetricsTracker | |||||
| from modelscope.trainers.nlp.space.metrics.metrics_tracker import \ | |||||
| MetricsTracker | |||||
| def get_logger(log_path, name='default'): | def get_logger(log_path, name='default'): | ||||