diff --git a/docs/source/quick_start.md b/docs/source/quick_start.md index 4a76f690..4d145e79 100644 --- a/docs/source/quick_start.md +++ b/docs/source/quick_start.md @@ -101,7 +101,7 @@ import cv2 import os.path as osp from maas_lib.pipelines import pipeline from maas_lib.utils.constant import Tasks -from pydatasets import PyDataset +from maas_lib.pydatasets import PyDataset # 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 input_location = [ diff --git a/maas_lib/pipelines/base.py b/maas_lib/pipelines/base.py index 47c6d90b..76747b05 100644 --- a/maas_lib/pipelines/base.py +++ b/maas_lib/pipelines/base.py @@ -2,14 +2,14 @@ import os.path as osp from abc import ABC, abstractmethod -from typing import Any, Dict, Generator, List, Tuple, Union +from typing import Any, Dict, Generator, List, Union from maas_hub.snapshot_download import snapshot_download -from pydatasets import PyDataset from maas_lib.models import Model from maas_lib.pipelines import util from maas_lib.preprocessors import Preprocessor +from maas_lib.pydatasets import PyDataset from maas_lib.utils.config import Config from .util import is_model_name diff --git a/pydatasets/__init__.py b/maas_lib/pydatasets/__init__.py similarity index 100% rename from pydatasets/__init__.py rename to maas_lib/pydatasets/__init__.py diff --git a/pydatasets/py_dataset.py b/maas_lib/pydatasets/py_dataset.py similarity index 95% rename from pydatasets/py_dataset.py rename to maas_lib/pydatasets/py_dataset.py index 2e9a378f..58f83830 100644 --- a/pydatasets/py_dataset.py +++ b/maas_lib/pydatasets/py_dataset.py @@ -11,7 +11,7 @@ logger = get_logger() class PyDataset: _hf_ds = None # holds the underlying HuggingFace Dataset - """A PyDataset backed by hugging face datasets.""" + """A PyDataset backed by hugging face Dataset.""" def __init__(self, hf_ds: Dataset): self._hf_ds = hf_ds @@ -52,7 +52,7 @@ class PyDataset: Mapping[str, Union[str, Sequence[str]]]]] = None ) -> 'PyDataset': - """Load a pydataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset. + """Load a PyDataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset. Args: path (str): Path or name of the dataset. @@ -64,7 +64,7 @@ class PyDataset: split (str, optional): Which split of the data to load. Returns: - pydataset (obj:`PyDataset`): PyDataset object for a certain dataset. + PyDataset (obj:`PyDataset`): PyDataset object for a certain dataset. """ if isinstance(path, str): dataset = load_dataset( diff --git a/tests/pipelines/test_image_matting.py b/tests/pipelines/test_image_matting.py index 33e8c28c..8153b70d 100644 --- a/tests/pipelines/test_image_matting.py +++ b/tests/pipelines/test_image_matting.py @@ -5,10 +5,10 @@ import tempfile import unittest import cv2 -from pydatasets import PyDataset from maas_lib.fileio import File from maas_lib.pipelines import pipeline, util +from maas_lib.pydatasets import PyDataset from maas_lib.utils.constant import Tasks diff --git a/tests/pipelines/test_text_classification.py b/tests/pipelines/test_text_classification.py index f599b205..b6528319 100644 --- a/tests/pipelines/test_text_classification.py +++ b/tests/pipelines/test_text_classification.py @@ -1,17 +1,15 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -import os import shutil import unittest import zipfile from pathlib import Path -from pydatasets import PyDataset - from maas_lib.fileio import File from maas_lib.models import Model from maas_lib.models.nlp import BertForSequenceClassification from maas_lib.pipelines import SequenceClassificationPipeline, pipeline, util from maas_lib.preprocessors import SequenceClassificationPreprocessor +from maas_lib.pydatasets import PyDataset from maas_lib.utils.constant import Tasks diff --git a/tests/pydataset/__init__.py b/tests/pydatasets/__init__.py similarity index 100% rename from tests/pydataset/__init__.py rename to tests/pydatasets/__init__.py diff --git a/tests/pydataset/test_py_dataset.py b/tests/pydatasets/test_py_dataset.py similarity index 92% rename from tests/pydataset/test_py_dataset.py rename to tests/pydatasets/test_py_dataset.py index f6bdb8e9..a32dcb0e 100644 --- a/tests/pydataset/test_py_dataset.py +++ b/tests/pydatasets/test_py_dataset.py @@ -1,13 +1,14 @@ import unittest import datasets as hfdata -from pydatasets import PyDataset + +from maas_lib.pydatasets import PyDataset class PyDatasetTest(unittest.TestCase): def setUp(self): - # ds1 initiazed from in memory json + # ds1 initialized from in memory json self.json_data = { 'dummy': [{ 'a': i,