diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index dc4d0ab2..2aab142e 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -390,11 +390,13 @@ class HubApi: return resp['Data'] def list_oss_dataset_objects(self, dataset_name, namespace, max_limit, - is_recursive, is_filter_dir, revision, - cookies): + is_recursive, is_filter_dir, revision): url = f'{self.endpoint}/api/v1/datasets/{namespace}/{dataset_name}/oss/tree/?' \ f'MaxLimit={max_limit}&Revision={revision}&Recursive={is_recursive}&FilterDir={is_filter_dir}' - cookies = requests.utils.dict_from_cookiejar(cookies) + + cookies = ModelScopeConfig.get_cookies() + if cookies: + cookies = requests.utils.dict_from_cookiejar(cookies) resp = requests.get(url=url, cookies=cookies) resp = resp.json() diff --git a/modelscope/msdatasets/utils/dataset_utils.py b/modelscope/msdatasets/utils/dataset_utils.py index db9d1fee..c7aa7682 100644 --- a/modelscope/msdatasets/utils/dataset_utils.py +++ b/modelscope/msdatasets/utils/dataset_utils.py @@ -7,7 +7,7 @@ from typing import Any, Mapping, Optional, Sequence, Union from datasets.builder import DatasetBuilder from modelscope.hub.api import HubApi -from modelscope.utils.constant import DEFAULT_DATASET_REVISION, DownloadParams +from modelscope.utils.constant import DEFAULT_DATASET_REVISION from modelscope.utils.logger import get_logger from .dataset_builder import MsCsvDatasetBuilder, TaskSpecificDatasetBuilder @@ -95,15 +95,13 @@ def list_dataset_objects(hub_api: HubApi, max_limit: int, is_recursive: bool, res (list): List of objects, i.e., ['train/images/001.png', 'train/images/002.png', 'val/images/001.png', ...] """ res = [] - cookies = hub_api.check_cookies_upload_data(use_cookies=True) objects = hub_api.list_oss_dataset_objects( dataset_name=dataset_name, namespace=namespace, max_limit=max_limit, is_recursive=is_recursive, is_filter_dir=True, - revision=version, - cookies=cookies) + revision=version) for item in objects: object_key = item.get('Key') @@ -174,7 +172,7 @@ def get_dataset_files(subset_split_into: dict, modelscope_api = HubApi() objects = list_dataset_objects( hub_api=modelscope_api, - max_limit=DownloadParams.MAX_LIST_OBJECTS_NUM.value, + max_limit=-1, is_recursive=True, dataset_name=dataset_name, namespace=namespace, diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 6ba58c19..6c0f3e98 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -231,13 +231,6 @@ class DownloadMode(enum.Enum): FORCE_REDOWNLOAD = 'force_redownload' -class DownloadParams(enum.Enum): - """ - Parameters for downloading dataset. - """ - MAX_LIST_OBJECTS_NUM = 50000 - - class DatasetFormations(enum.Enum): """ How a dataset is organized and interpreted """