Overwrite default port/hostname from env variables

update copyright

add examples to CacheClient docstring

mindspore/ccsrc/minddata/dataset/core/config_manager.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,9 +57,8 @@ ConfigManager::ConfigManager()
     char *end = nullptr;
     cache_port_ = strtol(env_cache_port, &end, 10);
     if (*end != '\0') {
-      MS_LOG(WARNING) << "\nCache port from env variable MS_CACHE_PORT is invalid, back to use default "
-                      << kCfgDefaultCachePort << std::endl;
-      cache_port_ = kCfgDefaultCachePort;
+      MS_LOG(WARNING) << "Cache port from env variable MS_CACHE_PORT is invalid\n";
+      cache_port_ = 0;  // cause the port range validation to generate an error during the validation checks
     }
   }
 }

mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,6 +50,19 @@ CacheAdminArgHandler::CacheAdminArgHandler()
       hostname_(kCfgDefaultCacheHost),
       spill_dir_(""),
       command_id_(CommandId::kCmdUnknown) {
+  const char *env_cache_host = std::getenv("MS_CACHE_HOST");
+  const char *env_cache_port = std::getenv("MS_CACHE_PORT");
+  if (env_cache_host != nullptr) {
+    hostname_ = env_cache_host;
+  }
+  if (env_cache_port != nullptr) {
+    char *end = nullptr;
+    port_ = strtol(env_cache_port, &end, 10);
+    if (*end != '\0') {
+      std::cerr << "Cache port from env variable MS_CACHE_PORT is invalid\n";
+      port_ = 0;  // cause the port range validation to generate an error during the validation checks
+    }
+  }
   // Initialize the command mappings
   arg_map_["-h"] = ArgValue::kArgHost;
   arg_map_["--hostname"] = ArgValue::kArgHost;

mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -307,7 +307,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) {
         if (rc == StatusCode::kMDNetWorkError && retry_count < max_retries) {
           // If we get some network error, we will attempt some retries
           retry_count++;
-        } else if (rc.IsError()) {
+        } else if (rc.IsError() && rc.StatusCode() != StatusCode::kMDInterrupted) {
           MS_LOG(WARNING) << rc.ToString();
           return rc;
         }

mindspore/dataset/engine/cache_client.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Huawei Technologies Co., Ltd
+# Copyright 2019-2021 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,6 +38,14 @@ class DatasetCache:
         num_connections (int, optional): Number of tcp/ip connections (default=12).
         prefetch_size (int, optional): Prefetch size (default=20).
 
+    Examples:
+            >>> import mindspore.dataset as ds
+            >>>
+            >>> # create a cache instance, in which session_id is generated from command line `cache_admin -g`
+            >>> some_cache = ds.DatasetCache(session_id=session_id, size=0)
+            >>>
+            >>> dataset_dir = "path/to/imagefolder_directory"
+            >>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache)
     """
 
     def __init__(self, session_id, size=0, spilling=False, hostname=None, port=None, num_connections=None,

tests/ut/python/dataset/test_cache_map.py

@@ -2110,11 +2110,59 @@ def test_cache_map_nested_repeat():
 
 
 if __name__ == '__main__':
+    # This is just a list of tests, don't try to run these tests with 'python test_cache_map.py'
+    # since cache server is required to be brought up first
     test_cache_map_basic1()
     test_cache_map_basic2()
     test_cache_map_basic3()
     test_cache_map_basic4()
+    test_cache_map_basic5()
     test_cache_map_failure1()
     test_cache_map_failure2()
     test_cache_map_failure3()
     test_cache_map_failure4()
+    test_cache_map_failure5()
+    test_cache_map_failure6()
+    test_cache_map_failure7()
+    test_cache_map_failure8()
+    test_cache_map_failure9()
+    test_cache_map_failure10()
+    test_cache_map_failure11()
+    test_cache_map_split1()
+    test_cache_map_split2()
+    test_cache_map_parameter_check()
+    test_cache_map_running_twice1()
+    test_cache_map_running_twice2()
+    test_cache_map_extra_small_size1()
+    test_cache_map_extra_small_size2()
+    test_cache_map_no_image()
+    test_cache_map_parallel_pipeline1(shard=0)
+    test_cache_map_parallel_pipeline2(shard=1)
+    test_cache_map_parallel_workers()
+    test_cache_map_server_workers_1()
+    test_cache_map_server_workers_100()
+    test_cache_map_num_connections_1()
+    test_cache_map_num_connections_100()
+    test_cache_map_prefetch_size_1()
+    test_cache_map_prefetch_size_100()
+    test_cache_map_to_device()
+    test_cache_map_epoch_ctrl1()
+    test_cache_map_epoch_ctrl2()
+    test_cache_map_epoch_ctrl3()
+    test_cache_map_coco1()
+    test_cache_map_coco2()
+    test_cache_map_mnist1()
+    test_cache_map_mnist2()
+    test_cache_map_celeba1()
+    test_cache_map_celeba2()
+    test_cache_map_manifest1()
+    test_cache_map_manifest2()
+    test_cache_map_cifar1()
+    test_cache_map_cifar2()
+    test_cache_map_cifar3()
+    test_cache_map_cifar4()
+    test_cache_map_voc1()
+    test_cache_map_voc2()
+    test_cache_map_python_sampler1()
+    test_cache_map_python_sampler2()
+    test_cache_map_nested_repeat()

tests/ut/python/dataset/test_cache_nomap.py

@@ -2125,6 +2125,8 @@ def test_cache_nomap_failure5():
 
 
 if __name__ == '__main__':
+    # This is just a list of tests, don't try to run these tests with 'python test_cache_nomap.py'
+    # since cache server is required to be brought up first
     test_cache_nomap_basic1()
     test_cache_nomap_basic2()
     test_cache_nomap_basic3()
@@ -2132,8 +2134,49 @@ if __name__ == '__main__':
     test_cache_nomap_basic5()
     test_cache_nomap_basic6()
     test_cache_nomap_basic7()
+    test_cache_nomap_basic8()
+    test_cache_nomap_basic9()
     test_cache_nomap_allowed_share1()
     test_cache_nomap_allowed_share2()
     test_cache_nomap_allowed_share3()
     test_cache_nomap_allowed_share4()
     test_cache_nomap_disallowed_share1()
+    test_cache_nomap_running_twice1()
+    test_cache_nomap_running_twice2()
+    test_cache_nomap_extra_small_size1()
+    test_cache_nomap_extra_small_size2()
+    test_cache_nomap_parallel_pipeline1(shard=0)
+    test_cache_nomap_parallel_pipeline2(shard=1)
+    test_cache_nomap_parallel_workers()
+    test_cache_nomap_server_workers_1()
+    test_cache_nomap_server_workers_100()
+    test_cache_nomap_num_connections_1()
+    test_cache_nomap_num_connections_100()
+    test_cache_nomap_prefetch_size_1()
+    test_cache_nomap_prefetch_size_100()
+    test_cache_nomap_to_device()
+    test_cache_nomap_session_destroy()
+    test_cache_nomap_server_stop()
+    test_cache_nomap_epoch_ctrl1()
+    test_cache_nomap_epoch_ctrl2()
+    test_cache_nomap_epoch_ctrl3()
+    test_cache_nomap_epoch_ctrl4()
+    test_cache_nomap_multiple_cache1()
+    test_cache_nomap_multiple_cache2()
+    test_cache_nomap_multiple_cache3()
+    test_cache_nomap_multiple_cache_train()
+    test_cache_nomap_multiple_cache_eval()
+    test_cache_nomap_clue1()
+    test_cache_nomap_clue2()
+    test_cache_nomap_csv1()
+    test_cache_nomap_csv2()
+    test_cache_nomap_textfile1()
+    test_cache_nomap_textfile2()
+    test_cache_nomap_nested_repeat()
+    test_cache_nomap_get_repeat_count()
+    test_cache_nomap_long_file_list()
+    test_cache_nomap_failure1()
+    test_cache_nomap_failure2()
+    test_cache_nomap_failure3()
+    test_cache_nomap_failure4()
+    test_cache_nomap_failure5()