From 8a65b0b1ff19bb325c0926a24ddd6c5d34e0e384 Mon Sep 17 00:00:00 2001 From: "yingda.chen" Date: Fri, 8 Jul 2022 14:52:29 +0800 Subject: [PATCH] [to #42322933]update docs Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9316720 --- docs/source/api/modelscope.hub.rst | 50 +++++++++++++ docs/source/api/modelscope.hub.utils.rst | 26 +++++++ docs/source/api/modelscope.models.nlp.rst | 72 +++++++++++++++++-- docs/source/api/modelscope.models.rst | 2 + ...datasets.rst => modelscope.msdatasets.rst} | 0 .../api/modelscope.pipelines.multi_modal.rst | 28 +++++++- docs/source/api/modelscope.pipelines.rst | 31 ++++++-- docs/source/api/modelscope.rst | 1 + docs/source/api/modelscope.utils.rst | 8 --- docs/source/api/modules.rst | 7 -- docs/source/index.rst | 6 +- modelscope/hub/repository.py | 12 ++-- modelscope/models/nlp/__init__.py | 2 +- ...d_language_model.py => masked_language.py} | 0 modelscope/msdatasets/ms_dataset.py | 12 +++- modelscope/pipelines/builder.py | 4 +- .../pipelines/nlp/fill_mask_pipeline.py | 2 +- modelscope/preprocessors/image.py | 5 +- .../nlp/sequence_classification_trainer.py | 5 +- 19 files changed, 224 insertions(+), 49 deletions(-) create mode 100644 docs/source/api/modelscope.hub.rst create mode 100644 docs/source/api/modelscope.hub.utils.rst rename docs/source/api/{modelscope.pydatasets.rst => modelscope.msdatasets.rst} (100%) delete mode 100644 docs/source/api/modules.rst rename modelscope/models/nlp/{masked_language_model.py => masked_language.py} (100%) diff --git a/docs/source/api/modelscope.hub.rst b/docs/source/api/modelscope.hub.rst new file mode 100644 index 00000000..47d210c2 --- /dev/null +++ b/docs/source/api/modelscope.hub.rst @@ -0,0 +1,50 @@ +modelscope.hub package +========================= + +.. automodule:: modelscope.hub + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + modelscope.hub.utils + +Submodules +---------- + +modelscope.hub.api module +----------------------------- + +.. automodule:: modelscope.hub.api + :members: + :undoc-members: + :show-inheritance: + +modelscope.hub.git module +--------------------------- + +.. automodule:: modelscope.hub.git + :members: + :undoc-members: + :show-inheritance: + +modelscope.hub.file_download module +--------------------------- + +.. automodule:: modelscope.hub.file_download + :members: + :undoc-members: + :show-inheritance: + +modelscope.hub.snapshot_download module +--------------------------- + +.. automodule:: modelscope.hub.snapshot_download + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/modelscope.hub.utils.rst b/docs/source/api/modelscope.hub.utils.rst new file mode 100644 index 00000000..74d8ae96 --- /dev/null +++ b/docs/source/api/modelscope.hub.utils.rst @@ -0,0 +1,26 @@ +modelscope.hub.utils package +=============================== + +.. automodule:: modelscope.hub.utils + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +modelscope.hub.utils.caching module +------------------------------------------------------- + +.. automodule:: modelscope.hub.utils.caching + :members: + :undoc-members: + :show-inheritance: + +modelscope.pipelines.cv.image\_matting\_pipeline module +------------------------------------------------------- + +.. automodule:: modelscope.hub.utils.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/modelscope.models.nlp.rst b/docs/source/api/modelscope.models.nlp.rst index f332aca8..6cc411d4 100644 --- a/docs/source/api/modelscope.models.nlp.rst +++ b/docs/source/api/modelscope.models.nlp.rst @@ -9,18 +9,82 @@ modelscope.models.nlp package Submodules ---------- -modelscope.models.nlp.sequence\_classification\_model module +modelscope.models.nlp.bert\_for\_sequence\_classification module ------------------------------------------------------------ -.. automodule:: modelscope.models.nlp.sequence_classification_model +.. automodule:: modelscope.models.nlp.bert_for_sequence_classification :members: :undoc-members: :show-inheritance: -modelscope.models.nlp.text\_generation\_model module +modelscope.models.nlp.palm\_for\_text\_generation module ---------------------------------------------------- -.. automodule:: modelscope.models.nlp.text_generation_model +.. automodule:: modelscope.models.nlp.palm_for_text_generation + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.csanmt\_for\_translation module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.palm_for_text_generation + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.masked\_language module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.masked_language + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.sbert\_for\_nil module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.sbert_for_nil + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.sbert\_for\_sentence\_similarity module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.sbert_for_sentence_similarity + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.sbert\_for\_sentiment\_classification module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.sbert_for_sentiment_classification + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.sbert\_for\_sequence\_classification module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.sbert_for_sequence_classification + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.sbert\_for\_token\_classification module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.sbert_for_token_classification + :members: + :undoc-members: + :show-inheritance: + +modelscope.models.nlp.sbert\_for\_zero\_shot\_classification module +---------------------------------------------------- + +.. automodule:: modelscope.models.nlp.sbert_for_zero_shot_classification :members: :undoc-members: :show-inheritance: diff --git a/docs/source/api/modelscope.models.rst b/docs/source/api/modelscope.models.rst index 8f2870b3..2eaa1a6b 100644 --- a/docs/source/api/modelscope.models.rst +++ b/docs/source/api/modelscope.models.rst @@ -14,6 +14,8 @@ Subpackages modelscope.models.cv modelscope.models.nlp + modelscope.models.multi_modal + modelscope.models.audio Submodules ---------- diff --git a/docs/source/api/modelscope.pydatasets.rst b/docs/source/api/modelscope.msdatasets.rst similarity index 100% rename from docs/source/api/modelscope.pydatasets.rst rename to docs/source/api/modelscope.msdatasets.rst diff --git a/docs/source/api/modelscope.pipelines.multi_modal.rst b/docs/source/api/modelscope.pipelines.multi_modal.rst index 36df1c7c..4bc3982f 100644 --- a/docs/source/api/modelscope.pipelines.multi_modal.rst +++ b/docs/source/api/modelscope.pipelines.multi_modal.rst @@ -9,10 +9,34 @@ modelscope.pipelines.multi\_modal package Submodules ---------- -modelscope.pipelines.multi\_modal.image\_captioning module +modelscope.pipelines.multi\_modal.image\_captioning\_pipeline module ---------------------------------------------------------- -.. automodule:: modelscope.pipelines.multi_modal.image_captioning +.. automodule:: modelscope.pipelines.multi_modal.image_captioning_pipeline + :members: + :undoc-members: + :show-inheritance: + +modelscope.pipelines.multi\_modal.multi\_modal\_embedding\_pipeline module +---------------------------------------------------------- + +.. automodule:: modelscope.pipelines.multi_modal.multi_modal_embedding_pipeline + :members: + :undoc-members: + :show-inheritance: + +modelscope.pipelines.multi\_modal.text\_to\_image\_synthesis\_pipeline module +---------------------------------------------------------- + +.. automodule:: modelscope.pipelines.multi_modal.text_to_image_synthesis_pipeline + :members: + :undoc-members: + :show-inheritance: + +modelscope.pipelines.multi\_modal.visual\_question\_answering\_pipeline module +---------------------------------------------------------- + +.. automodule:: modelscope.pipelines.multi_modal.visual_question_answering_pipeline :members: :undoc-members: :show-inheritance: diff --git a/docs/source/api/modelscope.pipelines.rst b/docs/source/api/modelscope.pipelines.rst index e5758046..3e6c2694 100644 --- a/docs/source/api/modelscope.pipelines.rst +++ b/docs/source/api/modelscope.pipelines.rst @@ -1,28 +1,45 @@ modelscope.pipelines package ============================ +.. automodule:: modelscope.pipelines + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + modelscope.pipelines.cv + modelscope.pipelines.nlp + modelscope.pipelines.multi_modal + modelscope.pipelines.audio Submodules ---------- -modelscope.pipelines.base module --------------------------------- +modelscope.pipelines.builder module +----------------------------------- -.. automodule:: modelscope.pipelines.base +.. automodule:: modelscope.pipelines.builder :members: + :undoc-members: + :show-inheritance: -modelscope.pipelines.builder module +modelscope.pipelines.base module ----------------------------------- -.. automodule:: modelscope.pipelines.builder +.. automodule:: modelscope.pipelines.base :members: :undoc-members: :show-inheritance: -modelscope.pipelines.default module +modelscope.pipelines.outputs module ----------------------------------- -.. automodule:: modelscope.pipelines.default +.. automodule:: modelscope.pipelines.outputs :members: :undoc-members: :show-inheritance: diff --git a/docs/source/api/modelscope.rst b/docs/source/api/modelscope.rst index eacdf33d..d38654a4 100644 --- a/docs/source/api/modelscope.rst +++ b/docs/source/api/modelscope.rst @@ -19,6 +19,7 @@ Subpackages modelscope.msdatasets modelscope.trainers modelscope.utils + modelscope.hub Submodules ---------- diff --git a/docs/source/api/modelscope.utils.rst b/docs/source/api/modelscope.utils.rst index 0a78d4f4..3d705cfb 100644 --- a/docs/source/api/modelscope.utils.rst +++ b/docs/source/api/modelscope.utils.rst @@ -41,14 +41,6 @@ modelscope.utils.logger module :undoc-members: :show-inheritance: -modelscope.utils.pymod module ------------------------------ - -.. automodule:: modelscope.utils.pymod - :members: - :undoc-members: - :show-inheritance: - modelscope.utils.registry module -------------------------------- diff --git a/docs/source/api/modules.rst b/docs/source/api/modules.rst deleted file mode 100644 index 0f83e90c..00000000 --- a/docs/source/api/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -modelscope -========== - -.. toctree:: - :maxdepth: 4 - - modelscope diff --git a/docs/source/index.rst b/docs/source/index.rst index 11b6ee49..aba54341 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -29,7 +29,7 @@ ModelScope doc change_log.md -.. .. toctree:: +.. toctree:: .. :maxdepth: 10 .. :caption: API Doc @@ -38,6 +38,10 @@ ModelScope doc .. api/modelscope.pipelines .. api/modelscope.fileio .. api/modelscope.utils +.. api/modelscope.hub +.. api/modelscope.msdatasets +.. api/modelscope.tools +.. api/modelscope.trainers Indices and tables diff --git a/modelscope/hub/repository.py b/modelscope/hub/repository.py index b3ffbb22..6c112eb2 100644 --- a/modelscope/hub/repository.py +++ b/modelscope/hub/repository.py @@ -1,10 +1,9 @@ import os -from typing import List, Optional +from typing import Optional from modelscope.hub.errors import GitError, InvalidParameter from modelscope.utils.logger import get_logger from .api import ModelScopeConfig -from .constants import MODELSCOPE_URL_SCHEME from .git import GitCommandWrapper from .utils.utils import get_endpoint @@ -12,7 +11,7 @@ logger = get_logger() class Repository: - """Representation local model git repository. + """A local representation of the model git repository. """ def __init__( @@ -78,14 +77,15 @@ class Repository: def push(self, commit_message: str, branch: Optional[str] = 'master', - force: bool = False): - """Push local to remote, this method will do. + force: Optional[bool] = False): + """Push local files to remote, this method will do. git add git commit git push Args: commit_message (str): commit message - revision (Optional[str], optional): which branch to push. Defaults to 'master'. + branch (Optional[str]): which branch to push. Defaults to 'master'. + force (Optional[bool]): whether to use forced-push. """ if commit_message is None or not isinstance(commit_message, str): msg = 'commit_message must be provided!' diff --git a/modelscope/models/nlp/__init__.py b/modelscope/models/nlp/__init__.py index fb97fbb5..4f69a189 100644 --- a/modelscope/models/nlp/__init__.py +++ b/modelscope/models/nlp/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from modelscope.utils.error import TENSORFLOW_IMPORT_WARNING from .bert_for_sequence_classification import * # noqa F403 -from .masked_language_model import * # noqa F403 +from .masked_language import * # noqa F403 from .palm_for_text_generation import * # noqa F403 from .sbert_for_nli import * # noqa F403 from .sbert_for_sentence_similarity import * # noqa F403 diff --git a/modelscope/models/nlp/masked_language_model.py b/modelscope/models/nlp/masked_language.py similarity index 100% rename from modelscope/models/nlp/masked_language_model.py rename to modelscope/models/nlp/masked_language.py diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index fa7d1bf2..93d28ca7 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -28,8 +28,16 @@ def format_list(para) -> List: class MsDataset: - _hf_ds = None # holds the underlying HuggingFace Dataset - """A MsDataset backed by hugging face Dataset.""" + """ + ModelScope Dataset (aka, MsDataset) is backed by a huggingface Dataset to + provide efficient data access and local storage managements. On top of + that, MsDataset supports the data integration and interactions with multiple + remote hubs, particularly, ModelScope's own Dataset-hub. MsDataset also + abstracts away data-access details with other remote storage, including both + general external web-hosted data and cloud storage such as OSS. + """ + # the underlying huggingface Dataset + _hf_ds = None def __init__(self, hf_ds: Dataset, target: Optional[str] = None): self._hf_ds = hf_ds diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index ead7c521..61df971d 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -5,8 +5,8 @@ from typing import List, Optional, Union from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Pipelines from modelscope.models.base import Model -from modelscope.utils.config import Config, ConfigDict -from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.config import ConfigDict +from modelscope.utils.constant import Tasks from modelscope.utils.hub import read_config from modelscope.utils.registry import Registry, build_from_cfg from .base import Pipeline diff --git a/modelscope/pipelines/nlp/fill_mask_pipeline.py b/modelscope/pipelines/nlp/fill_mask_pipeline.py index bd4118cc..9ed48a30 100644 --- a/modelscope/pipelines/nlp/fill_mask_pipeline.py +++ b/modelscope/pipelines/nlp/fill_mask_pipeline.py @@ -5,7 +5,7 @@ import torch from ...metainfo import Pipelines from ...models import Model -from ...models.nlp.masked_language_model import MaskedLanguageModelBase +from ...models.nlp.masked_language import MaskedLanguageModelBase from ...preprocessors import FillMaskPreprocessor from ...utils.config import Config from ...utils.constant import ModelFile, Tasks diff --git a/modelscope/preprocessors/image.py b/modelscope/preprocessors/image.py index b2123fb7..fcad3c0e 100644 --- a/modelscope/preprocessors/image.py +++ b/modelscope/preprocessors/image.py @@ -18,9 +18,6 @@ class LoadImage: "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1). Args: mode (str): See :ref:`PIL.Mode`. - to_float32 (bool): Whether to convert the loaded image to a float32 - numpy array. If set to False, the loaded image is an uint8 array. - Defaults to False. """ def __init__(self, mode='rgb'): @@ -57,7 +54,7 @@ class LoadImage: return results def __repr__(self): - repr_str = (f'{self.__class__.__name__}(' f'mode={self.mode})') + repr_str = f'{self.__class__.__name__}(' f'mode={self.mode})' return repr_str diff --git a/modelscope/trainers/nlp/sequence_classification_trainer.py b/modelscope/trainers/nlp/sequence_classification_trainer.py index 7ae5576f..883110db 100644 --- a/modelscope/trainers/nlp/sequence_classification_trainer.py +++ b/modelscope/trainers/nlp/sequence_classification_trainer.py @@ -1,15 +1,12 @@ import time -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np -from modelscope.utils.constant import Tasks from modelscope.utils.logger import get_logger from ..base import BaseTrainer from ..builder import TRAINERS -# __all__ = ["SequenceClassificationTrainer"] - PATH = None logger = get_logger(PATH)