|
|
@@ -488,7 +488,8 @@ class Dataset: |
|
|
parallel (default=None, the value from the configuration will be used). |
|
|
parallel (default=None, the value from the configuration will be used). |
|
|
python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes. This |
|
|
python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes. This |
|
|
option could be beneficial if the Python operation is computational heavy (default=False). |
|
|
option could be beneficial if the Python operation is computational heavy (default=False). |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
callbacks: (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called (Default=None). |
|
|
callbacks: (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called (Default=None). |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -2241,7 +2242,8 @@ class MapDataset(Dataset): |
|
|
in parallel (default=None). |
|
|
in parallel (default=None). |
|
|
python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This |
|
|
python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This |
|
|
option could be beneficial if the Python operation is computational heavy (default=False). |
|
|
option could be beneficial if the Python operation is computational heavy (default=False). |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
callbacks: (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called (Default=None) |
|
|
callbacks: (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called (Default=None) |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
@@ -2980,7 +2982,8 @@ class ImageFolderDataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
@@ -3128,7 +3131,8 @@ class MnistDataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
@@ -3829,7 +3833,8 @@ class TFRecordDataset(SourceDataset): |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
shard_equal_rows (bool, optional): Get equal rows for all shards(default=False). If shard_equal_rows |
|
|
shard_equal_rows (bool, optional): Get equal rows for all shards(default=False). If shard_equal_rows |
|
|
is false, number of rows of each shard may be not equal. |
|
|
is false, number of rows of each shard may be not equal. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Examples: |
|
|
Examples: |
|
|
>>> import mindspore.dataset as ds |
|
|
>>> import mindspore.dataset as ds |
|
|
@@ -4019,7 +4024,8 @@ class ManifestDataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
@@ -4182,7 +4188,8 @@ class Cifar10Dataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
@@ -4322,7 +4329,8 @@ class Cifar100Dataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
@@ -4403,7 +4411,8 @@ class RandomDataset(SourceDataset): |
|
|
num_samples (int): number of samples to draw from the total. (default=None, which means all rows) |
|
|
num_samples (int): number of samples to draw from the total. (default=None, which means all rows) |
|
|
num_parallel_workers (int, optional): Number of workers to read the data |
|
|
num_parallel_workers (int, optional): Number of workers to read the data |
|
|
(default=None, number set in the config). |
|
|
(default=None, number set in the config). |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset |
|
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset |
|
|
(default=None, expected order behavior shown in the table). |
|
|
(default=None, expected order behavior shown in the table). |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided |
|
|
@@ -4666,7 +4675,8 @@ class VOCDataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If xml of Annotations is an invalid format. |
|
|
RuntimeError: If xml of Annotations is an invalid format. |
|
|
@@ -4861,7 +4871,8 @@ class CocoDataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Raises: |
|
|
Raises: |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
RuntimeError: If sampler and shuffle are specified at the same time. |
|
|
@@ -5013,7 +5024,8 @@ class CelebADataset(MappableDataset): |
|
|
into (default=None). |
|
|
into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Examples: |
|
|
Examples: |
|
|
>>> import mindspore.dataset as ds |
|
|
>>> import mindspore.dataset as ds |
|
|
@@ -5125,7 +5137,8 @@ class CLUEDataset(SourceDataset): |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Examples: |
|
|
Examples: |
|
|
>>> import mindspore.dataset as ds |
|
|
>>> import mindspore.dataset as ds |
|
|
@@ -5358,7 +5371,8 @@ class CSVDataset(SourceDataset): |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Examples: |
|
|
Examples: |
|
|
@@ -5469,7 +5483,8 @@ class TextFileDataset(SourceDataset): |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). |
|
|
num_shards (int, optional): Number of shards that the dataset will be divided into (default=None). |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
shard_id (int, optional): The shard ID within num_shards (default=None). This |
|
|
argument can only be specified when num_shards is also specified. |
|
|
argument can only be specified when num_shards is also specified. |
|
|
cache (DatasetCache, optional): Tensor cache to use. (default=None which means no cache is used). |
|
|
|
|
|
|
|
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. |
|
|
|
|
|
(default=None which means no cache is used). |
|
|
|
|
|
|
|
|
Examples: |
|
|
Examples: |
|
|
>>> import mindspore.dataset as ds |
|
|
>>> import mindspore.dataset as ds |
|
|
|