Browse Source

Overwrite default port/hostname from env variables

update copyright

add examples to CacheClient docstring
tags/v1.2.0-rc1
Lixia Chen 4 years ago
parent
commit
97cd142f8d
13 changed files with 126 additions and 15 deletions
  1. +3
    -4
      mindspore/ccsrc/minddata/dataset/core/config_manager.cc
  2. +14
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc
  3. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h
  4. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc
  5. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc
  6. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h
  7. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc
  8. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h
  9. +1
    -1
      mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs
  10. +2
    -2
      mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc
  11. +9
    -1
      mindspore/dataset/engine/cache_client.py
  12. +48
    -0
      tests/ut/python/dataset/test_cache_map.py
  13. +43
    -0
      tests/ut/python/dataset/test_cache_nomap.py

+ 3
- 4
mindspore/ccsrc/minddata/dataset/core/config_manager.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -57,9 +57,8 @@ ConfigManager::ConfigManager()
char *end = nullptr;
cache_port_ = strtol(env_cache_port, &end, 10);
if (*end != '\0') {
MS_LOG(WARNING) << "\nCache port from env variable MS_CACHE_PORT is invalid, back to use default "
<< kCfgDefaultCachePort << std::endl;
cache_port_ = kCfgDefaultCachePort;
MS_LOG(WARNING) << "Cache port from env variable MS_CACHE_PORT is invalid\n";
cache_port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
}


+ 14
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd

* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -50,6 +50,19 @@ CacheAdminArgHandler::CacheAdminArgHandler()
hostname_(kCfgDefaultCacheHost),
spill_dir_(""),
command_id_(CommandId::kCmdUnknown) {
const char *env_cache_host = std::getenv("MS_CACHE_HOST");
const char *env_cache_port = std::getenv("MS_CACHE_PORT");
if (env_cache_host != nullptr) {
hostname_ = env_cache_host;
}
if (env_cache_port != nullptr) {
char *end = nullptr;
port_ = strtol(env_cache_port, &end, 10);
if (*end != '\0') {
std::cerr << "Cache port from env variable MS_CACHE_PORT is invalid\n";
port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
// Initialize the command mappings
arg_map_["-h"] = ArgValue::kArgHost;
arg_map_["--hostname"] = ArgValue::kArgHost;


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd

* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd

* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd

* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd

* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd

* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 2
- 2
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -307,7 +307,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) {
if (rc == StatusCode::kMDNetWorkError && retry_count < max_retries) {
// If we get some network error, we will attempt some retries
retry_count++;
} else if (rc.IsError()) {
} else if (rc.IsError() && rc.StatusCode() != StatusCode::kMDInterrupted) {
MS_LOG(WARNING) << rc.ToString();
return rc;
}


+ 9
- 1
mindspore/dataset/engine/cache_client.py View File

@@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -38,6 +38,14 @@ class DatasetCache:
num_connections (int, optional): Number of tcp/ip connections (default=12).
prefetch_size (int, optional): Prefetch size (default=20).

Examples:
>>> import mindspore.dataset as ds
>>>
>>> # create a cache instance, in which session_id is generated from command line `cache_admin -g`
>>> some_cache = ds.DatasetCache(session_id=session_id, size=0)
>>>
>>> dataset_dir = "path/to/imagefolder_directory"
>>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache)
"""

def __init__(self, session_id, size=0, spilling=False, hostname=None, port=None, num_connections=None,


+ 48
- 0
tests/ut/python/dataset/test_cache_map.py View File

@@ -2110,11 +2110,59 @@ def test_cache_map_nested_repeat():


if __name__ == '__main__':
# This is just a list of tests, don't try to run these tests with 'python test_cache_map.py'
# since cache server is required to be brought up first
test_cache_map_basic1()
test_cache_map_basic2()
test_cache_map_basic3()
test_cache_map_basic4()
test_cache_map_basic5()
test_cache_map_failure1()
test_cache_map_failure2()
test_cache_map_failure3()
test_cache_map_failure4()
test_cache_map_failure5()
test_cache_map_failure6()
test_cache_map_failure7()
test_cache_map_failure8()
test_cache_map_failure9()
test_cache_map_failure10()
test_cache_map_failure11()
test_cache_map_split1()
test_cache_map_split2()
test_cache_map_parameter_check()
test_cache_map_running_twice1()
test_cache_map_running_twice2()
test_cache_map_extra_small_size1()
test_cache_map_extra_small_size2()
test_cache_map_no_image()
test_cache_map_parallel_pipeline1(shard=0)
test_cache_map_parallel_pipeline2(shard=1)
test_cache_map_parallel_workers()
test_cache_map_server_workers_1()
test_cache_map_server_workers_100()
test_cache_map_num_connections_1()
test_cache_map_num_connections_100()
test_cache_map_prefetch_size_1()
test_cache_map_prefetch_size_100()
test_cache_map_to_device()
test_cache_map_epoch_ctrl1()
test_cache_map_epoch_ctrl2()
test_cache_map_epoch_ctrl3()
test_cache_map_coco1()
test_cache_map_coco2()
test_cache_map_mnist1()
test_cache_map_mnist2()
test_cache_map_celeba1()
test_cache_map_celeba2()
test_cache_map_manifest1()
test_cache_map_manifest2()
test_cache_map_cifar1()
test_cache_map_cifar2()
test_cache_map_cifar3()
test_cache_map_cifar4()
test_cache_map_voc1()
test_cache_map_voc2()
test_cache_map_python_sampler1()
test_cache_map_python_sampler2()
test_cache_map_nested_repeat()

+ 43
- 0
tests/ut/python/dataset/test_cache_nomap.py View File

@@ -2125,6 +2125,8 @@ def test_cache_nomap_failure5():


if __name__ == '__main__':
# This is just a list of tests, don't try to run these tests with 'python test_cache_nomap.py'
# since cache server is required to be brought up first
test_cache_nomap_basic1()
test_cache_nomap_basic2()
test_cache_nomap_basic3()
@@ -2132,8 +2134,49 @@ if __name__ == '__main__':
test_cache_nomap_basic5()
test_cache_nomap_basic6()
test_cache_nomap_basic7()
test_cache_nomap_basic8()
test_cache_nomap_basic9()
test_cache_nomap_allowed_share1()
test_cache_nomap_allowed_share2()
test_cache_nomap_allowed_share3()
test_cache_nomap_allowed_share4()
test_cache_nomap_disallowed_share1()
test_cache_nomap_running_twice1()
test_cache_nomap_running_twice2()
test_cache_nomap_extra_small_size1()
test_cache_nomap_extra_small_size2()
test_cache_nomap_parallel_pipeline1(shard=0)
test_cache_nomap_parallel_pipeline2(shard=1)
test_cache_nomap_parallel_workers()
test_cache_nomap_server_workers_1()
test_cache_nomap_server_workers_100()
test_cache_nomap_num_connections_1()
test_cache_nomap_num_connections_100()
test_cache_nomap_prefetch_size_1()
test_cache_nomap_prefetch_size_100()
test_cache_nomap_to_device()
test_cache_nomap_session_destroy()
test_cache_nomap_server_stop()
test_cache_nomap_epoch_ctrl1()
test_cache_nomap_epoch_ctrl2()
test_cache_nomap_epoch_ctrl3()
test_cache_nomap_epoch_ctrl4()
test_cache_nomap_multiple_cache1()
test_cache_nomap_multiple_cache2()
test_cache_nomap_multiple_cache3()
test_cache_nomap_multiple_cache_train()
test_cache_nomap_multiple_cache_eval()
test_cache_nomap_clue1()
test_cache_nomap_clue2()
test_cache_nomap_csv1()
test_cache_nomap_csv2()
test_cache_nomap_textfile1()
test_cache_nomap_textfile2()
test_cache_nomap_nested_repeat()
test_cache_nomap_get_repeat_count()
test_cache_nomap_long_file_list()
test_cache_nomap_failure1()
test_cache_nomap_failure2()
test_cache_nomap_failure3()
test_cache_nomap_failure4()
test_cache_nomap_failure5()

Loading…
Cancel
Save