!6450 Change PS directory.

Merge pull request !6450 from ZPaC/master-change-ps-dir

mindspore/ccsrc/CMakeLists.txt

@@ -161,7 +161,7 @@ set(SUB_COMP
         frontend/operator
         pipeline/jit
         pipeline/pynative
-        common debug pybind_api utils vm profiler
+        common debug pybind_api utils vm profiler ps
 )
 
 foreach (_comp ${SUB_COMP})

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/embedding_look_up_proxy_kernel.cc

@@ -15,7 +15,7 @@
  */
 #include "backend/kernel_compiler/cpu/ps/embedding_look_up_proxy_kernel.h"
 #include <vector>
-#include "frontend/parallel/ps/worker.h"
+#include "ps/worker.h"
 
 namespace mindspore {
 namespace kernel {
@@ -30,7 +30,7 @@ void EmbeddingLookUpProxyKernel::InitKernel(const CNodePtr &kernel_node) {
     input_dims_ *= dim;
   }
 
-  if (mindspore::parallel::ps::Util::IsRoleOfWorker()) {
+  if (mindspore::ps::Util::IsRoleOfWorker()) {
     key_ = AnfAlgo::GetNodeAttr<size_t>(kernel_node, kAttrPsKey);
   }
   std::vector<size_t> keys{key_, key_, key_};
@@ -41,9 +41,9 @@ void EmbeddingLookUpProxyKernel::InitKernel(const CNodePtr &kernel_node) {
   MS_LOG(INFO) << "Init embedding lookup proxy kernel, input shape:" << input_shape
                << ", indices_shape:" << indices_shape << ", output_shape:" << output_shape;
   std::vector<int> lens{SizeToInt(input_shape.size()), SizeToInt(indices_shape.size()), SizeToInt(output_shape.size())};
-  if (mindspore::parallel::ps::Util::IsRoleOfWorker()) {
-    parallel::ps::worker.AddEmbeddingTable(key_, input_shape[axis]);
-    parallel::ps::worker.InitPSEmbeddingTable(keys, values, lens);
+  if (mindspore::ps::Util::IsRoleOfWorker()) {
+    mindspore::ps::worker.AddEmbeddingTable(key_, input_shape[axis]);
+    mindspore::ps::worker.InitPSEmbeddingTable(keys, values, lens);
   }
 }
 
@@ -70,8 +70,8 @@ bool EmbeddingLookUpProxyKernel::Launch(const std::vector<kernel::AddressPtr> &i
     MS_LOG(EXCEPTION) << "Lookup id memcpy failed.";
     return false;
   }
-  parallel::ps::worker.DoPSEmbeddingLookup({key_}, lookup_ids, lengths, &lookup_result,
-                                           parallel::ps::kEmbeddingLookupCmd);
+  mindspore::ps::worker.DoPSEmbeddingLookup({key_}, lookup_ids, lengths, &lookup_result,
+                                            mindspore::ps::kEmbeddingLookupCmd);
 
   auto ret2 = memcpy_s(output_addr, outputs[0]->size, lookup_result.data(), output_size);
   if (ret2 != EOK) {

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/embedding_look_up_ps_kernel.cc

@@ -19,12 +19,12 @@
 #include <memory>
 #include <functional>
 #include "backend/kernel_compiler/common_utils.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 
 namespace mindspore {
 namespace kernel {
 namespace ps {
-using mindspore::parallel::ps::Util;
+using mindspore::ps::Util;
 constexpr int kAxis = 0;
 void EmbeddingLookUpPSKernel::InitKernel(
   const std::shared_ptr<std::vector<std::shared_ptr<std::vector<size_t>>>> &shapes) {

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/pserver_kernel.h

@@ -19,12 +19,12 @@
 #include <vector>
 #include <memory>
 #include "backend/kernel_compiler/kernel.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 
 namespace mindspore {
 namespace kernel {
 namespace ps {
-using mindspore::parallel::ps::Util;
+using mindspore::ps::Util;
 class PServerKernel {
  public:
   PServerKernel(size_t rank_id, size_t pserver_num, size_t worker_num)

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/pull_kernel.h

@@ -19,8 +19,8 @@
 
 #include <vector>
 #include <string>
-#include "frontend/parallel/ps/worker.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/worker.h"
+#include "ps/util.h"
 #include "backend/kernel_compiler/cpu/cpu_kernel.h"
 #include "backend/kernel_compiler/cpu/cpu_kernel_factory.h"
 
@@ -36,10 +36,10 @@ class PullKernel : public CPUKernel {
     if (inputs.size() != 2) {
       MS_LOG(EXCEPTION) << "Inputs size is " << inputs.size() << ", but PullKernel needs 2.";
     }
-    bool init_in_server = parallel::ps::worker.GetParamInitInServer(param_name_);
+    bool init_in_server = mindspore::ps::worker.GetParamInitInServer(param_name_);
     // If init_in_server, forward kernel should run in server too.
     if (!init_in_server) {
-      parallel::ps::Worker<T>::GetInstance().Pull(key_, inputs[1]->addr, inputs[1]->size);
+      mindspore::ps::worker.Pull(key_, inputs[1]->addr, inputs[1]->size);
     }
     return true;
   }
@@ -62,7 +62,7 @@ class PullKernel : public CPUKernel {
     MS_EXCEPTION_IF_NULL(param_node);
     param_name_ = param_node->fullname_with_scope();
 
-    if (mindspore::parallel::ps::Util::IsRoleOfWorker()) {
+    if (mindspore::ps::Util::IsRoleOfWorker()) {
       key_ = AnfAlgo::GetNodeAttr<size_t>(kernel_node, kAttrPsKey);
     }
     InitSizeLists();

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/push_kernel.h

@@ -19,8 +19,8 @@
 
 #include <vector>
 #include <algorithm>
-#include "frontend/parallel/ps/worker.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/worker.h"
+#include "ps/util.h"
 #include "backend/kernel_compiler/cpu/cpu_kernel.h"
 #include "backend/kernel_compiler/cpu/cpu_kernel_factory.h"
 
@@ -45,7 +45,7 @@ class PushKernel : public CPUKernel {
       addrs.push_back(reinterpret_cast<uintptr_t>(input->addr));
       sizes.push_back(SizeToInt(input->size) / sizeof(T));
     }
-    parallel::ps::Worker<T>::GetInstance().Push(keys, addrs, sizes);
+    mindspore::ps::worker.Push(keys, addrs, sizes);
     auto ret = memcpy_s(outputs[0]->addr, outputs[0]->size, &key_, sizeof(size_t));
     if (ret != EOK) {
       MS_LOG(EXCEPTION) << "Lookup id memcpy failed.";
@@ -62,7 +62,7 @@ class PushKernel : public CPUKernel {
     MS_LOG(INFO) << "Only init shape indices are " << only_shape_indices;
     for (size_t i = 0; i < optim_input_shapes.size(); i++) {
       auto shape = optim_input_shapes[i];
-      parallel::ps::worker.SetOptimInputShapes(key_, shape);
+      mindspore::ps::worker.SetOptimInputShapes(key_, shape);
       if (std::count(only_shape_indices.begin(), only_shape_indices.end(), i) == 0) {
         size_t size = sizeof(T);
         for (size_t j = 0; j < shape.size(); j++) {

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/sparse_apply_adam_ps_kernel.cc

@@ -17,7 +17,7 @@
 #include <memory>
 #include "backend/kernel_compiler/common_utils.h"
 #include "runtime/device/cpu/cpu_device_address.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 
 namespace mindspore {
 namespace kernel {

mindspore/ccsrc/backend/kernel_compiler/cpu/ps/sparse_apply_lazy_adam_ps_kernel.cc

@@ -17,7 +17,7 @@
 #include <memory>
 #include "backend/kernel_compiler/common_utils.h"
 #include "runtime/device/cpu/cpu_device_address.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 
 namespace mindspore {
 namespace kernel {

mindspore/ccsrc/backend/session/cpu_session.cc

@@ -27,7 +27,7 @@
 #include "backend/optimizer/common/pass_manager.h"
 #include "backend/optimizer/pass/replace_node_by_proxy.h"
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 #endif
 
 namespace mindspore {
@@ -69,7 +69,7 @@ GraphId CPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList
   SetKernelInfo(graph.get());
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
   AssignParamKey(graph);
-  if (parallel::ps::Util::IsRoleOfWorker()) {
+  if (ps::Util::IsRoleOfWorker()) {
     Optimize(graph);
   }
 #endif

mindspore/ccsrc/backend/session/session_basic.cc

@@ -36,9 +36,9 @@
 #include "ir/anf.h"
 #include "ir/func_graph_cloner.h"
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-#include "frontend/parallel/ps/worker.h"
-#include "frontend/parallel/ps/common.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/worker.h"
+#include "ps/common.h"
+#include "ps/util.h"
 #endif
 
 namespace mindspore {
@@ -1380,7 +1380,7 @@ void SessionBasic::RunGraphAsync(const GraphId &graph_id, const std::vector<tens
 
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 void SessionBasic::AssignParamKey(const KernelGraphPtr &kernel_graph) {
-  if (!parallel::ps::Util::IsRoleOfWorker()) {
+  if (!ps::Util::IsRoleOfWorker()) {
     MS_LOG(INFO) << "Not parameter server mode.";
     return;
   }
@@ -1393,7 +1393,7 @@ void SessionBasic::AssignParamKey(const KernelGraphPtr &kernel_graph) {
       if (AnfAlgo::GetCNodeName(node) == kEmbeddingLookupOpName) {
         size_t embedding_table_idx = 0;
         auto embedding_table = AnfAlgo::GetInputNode(node->cast<CNodePtr>(), embedding_table_idx);
-        size_t key = parallel::ps::worker.SetParamKey(embedding_table->fullname_with_scope());
+        size_t key = ps::worker.SetParamKey(embedding_table->fullname_with_scope());
         AnfAlgo::SetNodeAttr(kAttrPsKey, MakeValue(key), node);
       } else if (AnfAlgo::GetCNodeName(node) == kPushOpName) {
         auto pull_node = FindPullNode(node, node_list);
@@ -1404,12 +1404,12 @@ void SessionBasic::AssignParamKey(const KernelGraphPtr &kernel_graph) {
         // Second input of Pull node is the trainable parameter.
         size_t parameter_index = 1;
         auto parameter_node = AnfAlgo::GetInputNode(pull_node->cast<CNodePtr>(), parameter_index);
-        size_t key = parallel::ps::worker.SetParamKey(parameter_node->fullname_with_scope());
+        size_t key = ps::worker.SetParamKey(parameter_node->fullname_with_scope());
         AnfAlgo::SetNodeAttr(kAttrPsKey, MakeValue(key), node);
         AnfAlgo::SetNodeAttr(kAttrPsKey, MakeValue(key), pull_node);
 
         std::string optimizer_name = AnfAlgo::GetNodeAttr<std::string>(node, kAttrOptimizerType);
-        parallel::ps::worker.SetKeyOptimId(key, optimizer_name);
+        ps::worker.SetKeyOptimId(key, optimizer_name);
       }
     }
   }
@@ -1417,7 +1417,7 @@ void SessionBasic::AssignParamKey(const KernelGraphPtr &kernel_graph) {
 
 void SessionBasic::InitPSParamAndOptim(const KernelGraphPtr &kernel_graph,
                                        const std::vector<tensor::TensorPtr> &inputs_const) {
-  if (!parallel::ps::Util::IsRoleOfWorker()) {
+  if (!ps::Util::IsRoleOfWorker()) {
     return;
   }
   std::vector<tensor::TensorPtr> inputs(inputs_const);
@@ -1440,7 +1440,7 @@ void SessionBasic::InitPSParamAndOptim(const KernelGraphPtr &kernel_graph,
     MS_EXCEPTION_IF_NULL(input_node);
     if (input_node->isa<Parameter>() && AnfAlgo::OutputAddrExist(input_node, 0)) {
       auto pk_node = input_node->cast<ParameterPtr>();
-      parallel::ps::worker.InitPSParamAndOptim(pk_node->fullname_with_scope(), tensor);
+      ps::worker.InitPSParamAndOptim(pk_node->fullname_with_scope(), tensor);
     }
   }
 }

mindspore/ccsrc/frontend/parallel/CMakeLists.txt

@@ -1,12 +1,5 @@
 file(GLOB_RECURSE _PARALLEL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
 
-if (NOT (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)))
-    list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/optimizer_info_builder.cc")
-    list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/optimizer_info.cc")
-    list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/scheduler.cc")
-    list(REMOVE_ITEM _PARALLEL_SRC_FILES "ps/util.cc")
-endif()
-
 if (ENABLE_DUMP_PROTO)
     list(REMOVE_ITEM _PARALLEL_SRC_FILES "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc")
 endif ()

mindspore/ccsrc/pipeline/jit/action.cc

@@ -42,9 +42,9 @@
 #include "parse/python_adapter.h"
 #include "frontend/optimizer/py_pass_manager.h"
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-#include "frontend/parallel/ps/parameter_server.h"
-#include "frontend/parallel/ps/scheduler.h"
-#include "frontend/parallel/ps/worker.h"
+#include "ps/parameter_server.h"
+#include "ps/scheduler.h"
+#include "ps/worker.h"
 #endif
 
 namespace mindspore {
@@ -419,19 +419,19 @@ bool ExecuteAction(const ResourcePtr &res) {
 
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 bool StartPSWorkerAction(const ResourcePtr &res) {
-  parallel::ps::worker.Run();
+  ps::worker.Run();
   return true;
 }
 
 bool StartPSServerAction(const ResourcePtr &res) {
   FuncGraphPtr func_graph = res->func_graph();
-  auto &ps = parallel::ps::ParameterServer<float>::GetInstance();
+  auto &ps = ps::ParameterServer<float>::GetInstance();
   ps.Run(func_graph);
   return true;
 }
 
 bool StartPSSchedulerAction(const ResourcePtr &res) {
-  parallel::ps::Scheduler::GetInstance().Run();
+  ps::Scheduler::GetInstance().Run();
   return true;
 }
 #endif
@@ -585,7 +585,7 @@ std::vector<ActionItem> VmPipeline() {
 
   actions.emplace_back(std::make_pair("validate", ValidateAction));
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-  if (parallel::ps::Util::IsRoleOfWorker()) {
+  if (ps::Util::IsRoleOfWorker()) {
     actions.emplace_back(std::make_pair("worker", StartPSWorkerAction));
   }
 #endif

mindspore/ccsrc/pipeline/jit/init.cc

@@ -34,9 +34,9 @@
 #include "runtime/device/gpu/distribution/collective_fake_init.h"
 #endif
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 #endif
-#include "frontend/parallel/ps/ps_context.h"
+#include "ps/ps_context.h"
 
 #include "pybind_api/gil_scoped_long_running.h"
 
@@ -53,7 +53,7 @@ using OpInfoLoaderPy = mindspore::kernel::OpInfoLoaderPy;
 using ParallelContext = mindspore::parallel::ParallelContext;
 using CostModelContext = mindspore::parallel::CostModelContext;
 using mindspore::MsCtxParam;
-using PSContext = mindspore::parallel::ps::PSContext;
+using PSContext = mindspore::ps::PSContext;
 
 // Interface with python
 PYBIND11_MODULE(_c_expression, m) {

mindspore/ccsrc/pipeline/jit/pipeline.cc

@@ -47,9 +47,9 @@
 #include "pybind_api/pybind_patch.h"
 #include "utils/shape_utils.h"
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-#include "frontend/parallel/ps/common.h"
-#include "frontend/parallel/ps/util.h"
-#include "frontend/parallel/ps/worker.h"
+#include "ps/common.h"
+#include "ps/util.h"
+#include "ps/worker.h"
 #endif
 
 #if (ENABLE_GE || ENABLE_D)
@@ -413,14 +413,14 @@ std::vector<ActionItem> GetPipline(const ResourcePtr &resource, const std::strin
   std::string backend = MsContext::GetInstance()->backend_policy();
 
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-  if (mindspore::parallel::ps::Util::IsParamServerMode()) {
-    mindspore::parallel::ps::Util::SetInternalEnvVar();
+  if (mindspore::ps::Util::IsParamServerMode()) {
+    mindspore::ps::Util::SetInternalEnvVar();
   }
-  if (parallel::ps::Util::IsRoleOfPServer()) {
+  if (ps::Util::IsRoleOfPServer()) {
     resource->results()[kBackend] = compile::CreateBackend();
     return PServerPipeline();
   }
-  if (parallel::ps::Util::IsRoleOfScheduler()) {
+  if (ps::Util::IsRoleOfScheduler()) {
     return PSchedulerPipeline();
   }
 #endif
@@ -1002,9 +1002,9 @@ void ClearResAtexit() {
   pynative::ClearPyNativeSession();
   session::ClearPythonParasMap();
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
-  if (parallel::ps::Util::IsParamServerMode()) {
-    if (parallel::ps::Util::IsRoleOfWorker()) {
-      parallel::ps::worker.Finalize();
+  if (ps::Util::IsParamServerMode()) {
+    if (ps::Util::IsRoleOfWorker()) {
+      ps::worker.Finalize();
     }
   }
 #endif

mindspore/ccsrc/ps/CMakeLists.txt

@@ -0,0 +1,11 @@
+file(GLOB_RECURSE _PS_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
+
+if (NOT (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)))
+    list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info_builder.cc")
+    list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info.cc")
+    list(REMOVE_ITEM _PS_SRC_FILES "scheduler.cc")
+    list(REMOVE_ITEM _PS_SRC_FILES "util.cc")
+endif()
+
+set_property(SOURCE ${_PS_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PS)
+add_library(_mindspore_ps_obj OBJECT ${_PS_SRC_FILES})

mindspore/ccsrc/frontend/parallel/ps/common.h → mindspore/ccsrc/ps/common.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_COMMON_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_COMMON_H_
+#ifndef MINDSPORE_CCSRC_PS_COMMON_H_
+#define MINDSPORE_CCSRC_PS_COMMON_H_
 
 #include <iostream>
 #include <vector>
@@ -25,7 +25,6 @@
 #include "ps/ps.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 constexpr char kEnvCommType[] = "MS_COMM_TYPE";
 constexpr char kEnvInterface[] = "MS_INTERFACE";
@@ -131,6 +130,5 @@ const std::map<std::string, OptimOriginIdx> kOptimToPSSendIdx = {{kApplyMomentum
     }                                                                                           \
   }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_COMMON_H_
+#endif  // MINDSPORE_CCSRC_PS_COMMON_H_

mindspore/ccsrc/frontend/parallel/ps/optimizer_info.cc → mindspore/ccsrc/ps/optimizer_info.cc

@@ -14,15 +14,14 @@
  * limitations under the License.
  */
 
-#include "frontend/parallel/ps/optimizer_info.h"
+#include "ps/optimizer_info.h"
 #include <map>
 #include <memory>
 #include <string>
 #include <functional>
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 void OptimizerInfo::AddWorkspace(const AddressPtr &workspace) { workspaces_.push_back(workspace); }
 
@@ -367,5 +366,4 @@ size_t SparseFtrlOptimInfo::indices_index() {
   return ps_indices_index;
 }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore

mindspore/ccsrc/frontend/parallel/ps/optimizer_info.h → mindspore/ccsrc/ps/optimizer_info.h

@@ -14,16 +14,15 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_OPTIMIZER_INFO_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_OPTIMIZER_INFO_H_
+#ifndef MINDSPORE_CCSRC_PS_OPTIMIZER_INFO_H_
+#define MINDSPORE_CCSRC_PS_OPTIMIZER_INFO_H_
 
 #include <vector>
 #include <string>
 #include "backend/kernel_compiler/kernel.h"
-#include "frontend/parallel/ps/common.h"
+#include "ps/common.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 using mindspore::kernel::AddressPtr;
 class OptimizerInfo {
@@ -126,6 +125,5 @@ class SparseFtrlOptimInfo : public SparseOptimInfo {
   size_t indices_index() override;
 };
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_OPTIMIZER_INFO_H_
+#endif  // MINDSPORE_CCSRC_PS_OPTIMIZER_INFO_H_

mindspore/ccsrc/frontend/parallel/ps/optimizer_info_builder.cc → mindspore/ccsrc/ps/optimizer_info_builder.cc

@@ -14,14 +14,13 @@
  * limitations under the License.
  */
 
-#include "frontend/parallel/ps/optimizer_info_builder.h"
+#include "ps/optimizer_info_builder.h"
 #include <vector>
 #include <memory>
 #include <functional>
 #include "backend/kernel_compiler/cpu/ps/sparse_apply_ftrl_ps_kernel.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 using mindspore::kernel::ps::SparseApplyFtrlPSKernel;
 OptimizerInfo *OptimizerInfoBuilder::Build(const std::shared_ptr<PServerKernel> &pserver_kernel,
@@ -101,6 +100,7 @@ AddressPtr OptimizerInfoBuilder::GenInputAddrPtr(const std::string &optim_type,
   if (ret != 0) {
     MS_LOG(EXCEPTION) << "memcpy_s error, errorno(" << ret << ")";
     delete[] buffer;
+    buffer = nullptr;
     return nullptr;
   }
   return addr_ptr;
@@ -123,6 +123,7 @@ OptimizerInfo *MomentumOptimInfoBuilder::BuildInputs(const WeightPtr &weight, co
   if (ret != 0) {
     MS_LOG(EXCEPTION) << "memset_s error, errorno(" << ret << ")";
     delete[] reinterpret_cast<float *>(accumulate->addr);
+    accumulate->addr = nullptr;
     return nullptr;
   }
 
@@ -149,6 +150,7 @@ OptimizerInfo *SparseAdamOptimInfoBuilder::BuildInputs(const WeightPtr &weight,
   if (ret != 0) {
     MS_LOG(EXCEPTION) << "memset_s error, errorno(" << ret << ")";
     delete[] reinterpret_cast<float *>(m->addr);
+    m->addr = nullptr;
     return nullptr;
   }
 
@@ -161,7 +163,9 @@ OptimizerInfo *SparseAdamOptimInfoBuilder::BuildInputs(const WeightPtr &weight,
   if (ret != 0) {
     MS_LOG(EXCEPTION) << "memset_s error, errorno(" << ret << ")";
     delete[] reinterpret_cast<float *>(v->addr);
+    v->addr = nullptr;
     delete[] reinterpret_cast<float *>(m->addr);
+    m->addr = nullptr;
     return nullptr;
   }
 
@@ -205,6 +209,7 @@ OptimizerInfo *SparseFtrlOptimInfoBuilder::BuildInputs(const WeightPtr &weight,
   if (ret != 0) {
     MS_LOG(EXCEPTION) << "memset_s error, errorno(" << ret << ")";
     delete[] reinterpret_cast<float *>(linear->addr);
+    linear->addr = nullptr;
     return nullptr;
   }
   linear->size = weight->size() * sizeof(float);
@@ -214,5 +219,4 @@ OptimizerInfo *SparseFtrlOptimInfoBuilder::BuildInputs(const WeightPtr &weight,
   return new SparseFtrlOptimInfo(weight_addr, accum, linear, grad, indices);
 }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore

mindspore/ccsrc/frontend/parallel/ps/optimizer_info_builder.h → mindspore/ccsrc/ps/optimizer_info_builder.h

@@ -14,18 +14,17 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_OPTIMIZER_INFO_BUILDER_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_OPTIMIZER_INFO_BUILDER_H_
+#ifndef MINDSPORE_CCSRC_PS_OPTIMIZER_INFO_BUILDER_H_
+#define MINDSPORE_CCSRC_PS_OPTIMIZER_INFO_BUILDER_H_
 
 #include <vector>
 #include <memory>
 #include <string>
 #include "backend/kernel_compiler/kernel.h"
 #include "backend/kernel_compiler/cpu/ps/pserver_kernel.h"
-#include "frontend/parallel/ps/optimizer_info.h"
+#include "ps/optimizer_info.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 using mindspore::kernel::KernelMod;
 using mindspore::kernel::ps::PServerKernel;
@@ -55,6 +54,7 @@ class OptimizerInfoBuilder {
 class MomentumOptimInfoBuilder : public OptimizerInfoBuilder {
  public:
   explicit MomentumOptimInfoBuilder(size_t worker_num) : OptimizerInfoBuilder(worker_num) {}
+  ~MomentumOptimInfoBuilder() = default;
   OptimizerInfo *BuildInputs(const WeightPtr &weight, const Keys &keys, const Values &values, const Lengths &lens,
                              const InputsShapePtr &inputs_shape, size_t worker_num,
                              const std::shared_ptr<PServerKernel> &pserver_kernel) override;
@@ -63,6 +63,7 @@ class MomentumOptimInfoBuilder : public OptimizerInfoBuilder {
 class SparseAdamOptimInfoBuilder : public OptimizerInfoBuilder {
  public:
   explicit SparseAdamOptimInfoBuilder(size_t worker_num) : OptimizerInfoBuilder(worker_num) {}
+  ~SparseAdamOptimInfoBuilder() = default;
   OptimizerInfo *BuildInputs(const WeightPtr &weight, const Keys &keys, const Values &values, const Lengths &lens,
                              const InputsShapePtr &inputs_shape, size_t worker_num,
                              const std::shared_ptr<PServerKernel> &pserver_kernel) override;
@@ -71,11 +72,11 @@ class SparseAdamOptimInfoBuilder : public OptimizerInfoBuilder {
 class SparseFtrlOptimInfoBuilder : public OptimizerInfoBuilder {
  public:
   explicit SparseFtrlOptimInfoBuilder(size_t worker_num) : OptimizerInfoBuilder(worker_num) {}
+  ~SparseFtrlOptimInfoBuilder() = default;
   OptimizerInfo *BuildInputs(const WeightPtr &weight, const Keys &keys, const Values &values, const Lengths &lens,
                              const InputsShapePtr &inputs_shape, size_t worker_num,
                              const std::shared_ptr<PServerKernel> &pserver_kernel) override;
 };
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_OPTIMIZER_INFO_BUILDER_H_
+#endif  // MINDSPORE_CCSRC_PS_OPTIMIZER_INFO_BUILDER_H_

mindspore/ccsrc/frontend/parallel/ps/parameter_server.h → mindspore/ccsrc/ps/parameter_server.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_PARAMETER_SERVER_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_PARAMETER_SERVER_H_
+#ifndef MINDSPORE_CCSRC_PS_PARAMETER_SERVER_H_
+#define MINDSPORE_CCSRC_PS_PARAMETER_SERVER_H_
 
 #include <unistd.h>
 #include <unordered_map>
@@ -36,11 +36,11 @@
 #include "backend/session/session_basic.h"
 #include "backend/session/anf_runtime_algorithm.h"
 #include "backend/session/session_factory.h"
-#include "frontend/parallel/ps/common.h"
-#include "frontend/parallel/ps/optimizer_info.h"
-#include "frontend/parallel/ps/optimizer_info_builder.h"
-#include "frontend/parallel/ps/util.h"
-#include "frontend/parallel/ps/ps_context.h"
+#include "ps/common.h"
+#include "ps/optimizer_info.h"
+#include "ps/optimizer_info_builder.h"
+#include "ps/util.h"
+#include "ps/ps_context.h"
 #include "runtime/device/cpu/kernel_select_cpu.h"
 #include "utils/ms_context.h"
 #include "backend/kernel_compiler/kernel.h"
@@ -53,7 +53,6 @@
 #include "backend/kernel_compiler/cpu/ps/embedding_look_up_ps_kernel.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 using mindspore::kernel::ps::PServerKernel;
 using AnfAlgo = session::AnfRuntimeAlgorithm;
@@ -780,6 +779,5 @@ void ParameterServer<T>::Run(const FuncGraphPtr &func_graph) {
   MS_LOG(INFO) << "PServer finalized successfully.";
 }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_PARAMETER_SERVER_H_
+#endif  // MINDSPORE_CCSRC_PS_PARAMETER_SERVER_H_

mindspore/ccsrc/frontend/parallel/ps/ps_context.cc → mindspore/ccsrc/ps/ps_context.cc

@@ -14,12 +14,11 @@
  * limitations under the License.
  */
 
-#include "frontend/parallel/ps/ps_context.h"
+#include "ps/ps_context.h"
 #include "utils/log_adapter.h"
 #include "utils/ms_utils.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 std::shared_ptr<PSContext> PSContext::instance() {
   static std::shared_ptr<PSContext> ps_instance = nullptr;
@@ -82,5 +81,4 @@ void PSContext::SetPSRankId(int rank_id) { rank_id_ = rank_id; }
 
 int PSContext::ps_rank_id() const { return rank_id_; }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore

mindspore/ccsrc/frontend/parallel/ps/ps_context.h → mindspore/ccsrc/ps/ps_context.h

@@ -14,14 +14,13 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_CONTEXT_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_CONTEXT_H_
+#ifndef MINDSPORE_CCSRC_PS_CONTEXT_H_
+#define MINDSPORE_CCSRC_PS_CONTEXT_H_
 
 #include <string>
 #include <memory>
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 constexpr char kEnvRole[] = "MS_ROLE";
 constexpr char kEnvRoleOfPServer[] = "MS_PSERVER";
@@ -55,7 +54,6 @@ class PSContext {
   int rank_id_;
 };
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
 
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_CONTEXT_H_
+#endif  // MINDSPORE_CCSRC_PS_CONTEXT_H_

mindspore/ccsrc/frontend/parallel/ps/scheduler.cc → mindspore/ccsrc/ps/scheduler.cc

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 
-#include "frontend/parallel/ps/scheduler.h"
+#include "ps/scheduler.h"
 #include "ps/ps.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 void Scheduler::Run() {
   ::ps::Start(0);
@@ -26,5 +25,4 @@ void Scheduler::Run() {
   exit(1);
 }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore

mindspore/ccsrc/frontend/parallel/ps/scheduler.h → mindspore/ccsrc/ps/scheduler.h

@@ -14,10 +14,9 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_SCHEDULER_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_SCHEDULER_H_
+#ifndef MINDSPORE_CCSRC_PS_SCHEDULER_H_
+#define MINDSPORE_CCSRC_PS_SCHEDULER_H_
 namespace mindspore {
-namespace parallel {
 namespace ps {
 class Scheduler {
  public:
@@ -35,6 +34,5 @@ class Scheduler {
   Scheduler &operator=(const Scheduler &) = delete;
 };
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_SCHEDULER_H_
+#endif  // MINDSPORE_CCSRC_PS_SCHEDULER_H_

mindspore/ccsrc/frontend/parallel/ps/util.cc → mindspore/ccsrc/ps/util.cc

@@ -14,15 +14,14 @@
  * limitations under the License.
  */
 
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 #include <unordered_map>
 #include <vector>
-#include "frontend/parallel/ps/common.h"
-#include "frontend/parallel/ps/ps_context.h"
+#include "ps/common.h"
+#include "ps/ps_context.h"
 #include "utils/ms_utils.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 int Util::rank_id_ = -1;
 
@@ -58,19 +57,19 @@ bool Util::IsRoleOfScheduler() { return PSContext::instance()->is_role_sched();
 void Util::SetInternalEnvVar() {
   if (IsParamServerMode()) {
     auto comm_type = common::GetEnv(kEnvCommType);
-    if (comm_type.size() > 0) {
+    if (!comm_type.empty()) {
       (void)common::SetEnv(kDmlcCommType, comm_type.c_str());
     }
     auto interface = common::GetEnv(kEnvInterface);
-    if (interface.size() > 0) {
+    if (!interface.empty()) {
       (void)common::SetEnv(kDmlcInterface, interface.c_str());
     }
     auto server_num = common::GetEnv(kEnvPServerNum);
-    if (server_num.size() > 0) {
+    if (!server_num.empty()) {
       (void)common::SetEnv(kDmlcPServerNum, server_num.c_str());
     }
     auto worker_num = common::GetEnv(kEnvWorkerNum);
-    if (worker_num.size() > 0) {
+    if (!worker_num.empty()) {
       (void)common::SetEnv(kDmlcWorkerNum, worker_num.c_str());
     }
     if (IsRoleOfScheduler()) {
@@ -81,11 +80,11 @@ void Util::SetInternalEnvVar() {
       (void)common::SetEnv(kDmlcRole, kRoleOfWorker);
     }
     auto scheduler_host = common::GetEnv(kEnvSchedulerHost);
-    if (scheduler_host.size() > 0) {
+    if (!scheduler_host.empty()) {
       (void)common::SetEnv(kDmlcSchedulerHost, scheduler_host.c_str());
     }
     auto scheduler_port = common::GetEnv(kEnvSchedulerPort);
-    if (scheduler_port.size() > 0) {
+    if (!scheduler_port.empty()) {
       (void)common::SetEnv(kDmlcSchedulerPort, scheduler_port.c_str());
     }
   }
@@ -167,5 +166,4 @@ void Util::ReduceSparseGradient(float *gradients, int *indices, const size_t ind
   mindspore::kernel::SparseOptimizerCPUKernel::BucketReduceSparseGradient(param);
 }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore

mindspore/ccsrc/frontend/parallel/ps/util.h → mindspore/ccsrc/ps/util.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_UTIL_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_UTIL_H_
+#ifndef MINDSPORE_CCSRC_PS_UTIL_H_
+#define MINDSPORE_CCSRC_PS_UTIL_H_
 
 #include <map>
 #include <string>
@@ -25,7 +25,6 @@
 #include "backend/kernel_compiler/cpu/sparse_optimizer_cpu_kernel.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 class Util {
  public:
@@ -51,6 +50,5 @@ class Util {
   static int rank_id_;
 };
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_UTIL_H_
+#endif  // MINDSPORE_CCSRC_PS_UTIL_H_

mindspore/ccsrc/frontend/parallel/ps/worker.h → mindspore/ccsrc/ps/worker.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_WORKER_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_WORKER_H_
+#ifndef MINDSPORE_CCSRC_PS_WORKER_H_
+#define MINDSPORE_CCSRC_PS_WORKER_H_
 
 #include <utility>
 #include <memory>
@@ -27,13 +27,12 @@
 #include "ps/ps.h"
 #include "utils/log_adapter.h"
 #include "ir/tensor.h"
-#include "frontend/parallel/ps/util.h"
-#include "frontend/parallel/ps/common.h"
-#include "frontend/parallel/ps/worker_proxy.h"
+#include "ps/util.h"
+#include "ps/common.h"
+#include "ps/worker_proxy.h"
 #include "utils/shape_utils.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 template <typename T>
 class Worker {
@@ -363,6 +362,5 @@ void Worker<T>::AddEmbeddingTable(const ::ps::Key &key, const size_t &row_count)
 
 static Worker<float> &worker = Worker<float>::GetInstance();
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_WORKER_H_
+#endif  // MINDSPORE_CCSRC_PS_WORKER_H_

mindspore/ccsrc/frontend/parallel/ps/worker_proxy.h → mindspore/ccsrc/ps/worker_proxy.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_WORKER_PROXY_H_
-#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_WORKER_PROXY_H_
+#ifndef MINDSPORE_CCSRC_PS_WORKER_PROXY_H_
+#define MINDSPORE_CCSRC_PS_WORKER_PROXY_H_
 
 #include <map>
 #include <numeric>
@@ -27,12 +27,11 @@
 #include <memory>
 #include <vector>
 #include "ps/ps.h"
-#include "frontend/parallel/ps/util.h"
+#include "ps/util.h"
 #include "backend/kernel_compiler/common_utils.h"
-#include "frontend/parallel/ps/ps_context.h"
+#include "ps/ps_context.h"
 
 namespace mindspore {
-namespace parallel {
 namespace ps {
 template <typename T>
 class WorkerProxy : public ::ps::KVWorker<T> {
@@ -794,6 +793,5 @@ void WorkerProxy<T>::Send(::ps::Customer *customer, int timestamp, bool push, bo
   }
 }
 }  // namespace ps
-}  // namespace parallel
 }  // namespace mindspore
-#endif  // MINDSPORE_CCSRC_FRONTEND_PARALLEL_PS_WORKER_PROXY_H_
+#endif  // MINDSPORE_CCSRC_PS_WORKER_PROXY_H_

mindspore/context.py

@@ -666,14 +666,14 @@ def set_ps_context(**kwargs):
 
         .. code-block::
 
-        MS_SERVER_NUM  # Server number
-        MS_WORKER_NUM  # Worker number
-        MS_SCHED_HOST  # Scheduler IP address
-        MS_SCHED_PORT  # Scheduler port
-        MS_ROLE        # The role of this process:
-                       # MS_SCHED represents the scheduler,
-                       # MS_WORKER represents the worker,
-                       # MS_PSERVER represents the Server
+            MS_SERVER_NUM  # Server number
+            MS_WORKER_NUM  # Worker number
+            MS_SCHED_HOST  # Scheduler IP address
+            MS_SCHED_PORT  # Scheduler port
+            MS_ROLE        # The role of this process:
+                           # MS_SCHED represents the scheduler,
+                           # MS_WORKER represents the worker,
+                           # MS_PSERVER represents the Server
 
 
     Args:

mindspore/core/utils/log_adapter.cc

@@ -181,7 +181,8 @@ static const char *GetSubModuleName(SubModuleId module_id) {
     "SESSION",    // SM_SESSION
     "UTILS",      // SM_UTILS
     "VM",         // SM_VM
-    "PROFILER"    // SM_PROFILER
+    "PROFILER",   // SM_PROFILER
+    "PS"          // SM_PS
   };
 
   return sub_module_names[module_id % NUM_SUBMODUES];

mindspore/core/utils/log_adapter.h

@@ -125,6 +125,7 @@ enum SubModuleId : int {
   SM_UTILS,        // utils
   SM_VM,           // VM
   SM_PROFILER,     // profiler
+  SM_PS,           // Parameter Server
   NUM_SUBMODUES    // number of submodules
 };
 

mindspore/parallel/_ps_context.py

@@ -52,14 +52,17 @@ def _set_ps_context(**kwargs):
     Note:
         Some other environment variables should also be set for parameter server training mode.
         These environment variables are listed below:
-        MS_SERVER_NUM  # Server number
-        MS_WORKER_NUM  # Worker number
-        MS_SCHED_HOST  # Scheduler IP address
-        MS_SCHED_PORT  # Scheduler port
-        MS_ROLE        # The role of this process:
-                         MS_SCHED represents the scheduler,
-                         MS_WORKER represents the worker,
-                         MS_PSERVER represents the Server
+
+        .. code-block::
+
+            MS_SERVER_NUM  # Server number
+            MS_WORKER_NUM  # Worker number
+            MS_SCHED_HOST  # Scheduler IP address
+            MS_SCHED_PORT  # Scheduler port
+            MS_ROLE        # The role of this process:
+                           # MS_SCHED represents the scheduler,
+                           # MS_WORKER represents the worker,
+                           # MS_PSERVER represents the Server
 
 
     Args:

tests/ut/cpp/CMakeLists.txt

@@ -131,10 +131,10 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
         )
 
 list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/frontend/parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc")
-list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/frontend/parallel/ps/util.cc")
-list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/frontend/parallel/ps/scheduler.cc")
-list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/frontend/parallel/ps/optimizer_info.cc")
-list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/frontend/parallel/ps/optimizer_info_builder.cc")
+list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/ps/util.cc")
+list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/ps/scheduler.cc")
+list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/ps/optimizer_info.cc")
+list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/ps/optimizer_info_builder.cc")
 list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_add_relu_fusion.cc")
 list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_add_relu_grad_fusion.cc")
 list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_relu_fusion.cc")