!180 Serving, bugfix 0308

From: @xu-yfei
Reviewed-by: @zhangyinxia,@linqingke
Signed-off-by: @zhangyinxia

mindspore_serving/ccsrc/python/worker/worker_py.cc

@@ -46,6 +46,7 @@ void PyWorker::StartServable(const std::string &model_directory, const std::stri
   auto servable = std::make_shared<LocalModelServable>();
   auto status = servable->StartServable(model_directory, model_name, version_number);
   if (status != SUCCESS) {
+    servable->Clear();
     MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
   }
   status = Worker::GetInstance().StartServable(servable, notify_master);
@@ -72,6 +73,7 @@ void PyWorker::StartServableInMaster(const std::string &model_directory, const s
   auto servable = std::make_shared<LocalModelServable>();
   auto status = servable->StartServable(model_directory, model_name, version_number);
   if (status != SUCCESS) {
+    servable->Clear();
     MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
   }
   status = Worker::GetInstance().StartServable(servable, notify_master);
@@ -102,6 +104,7 @@ void PyWorker::StartDistributedServable(const std::string &servable_directory, c
   status = servable->StartServable(servable_directory, servable_name, rank_table_json_file, version_number,
                                    wait_agents_time_in_seconds);
   if (status != SUCCESS) {
+    servable->Clear();
     MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
   }
   status = Worker::GetInstance().StartServable(servable, notify_master);
@@ -131,6 +134,7 @@ void PyWorker::StartDistributedServableInMaster(const std::string &servable_dire
   status = servable->StartServable(servable_directory, servable_name, rank_table_json_file, version_number,
                                    wait_agents_time_in_seconds);
   if (status != SUCCESS) {
+    servable->Clear();
     MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage();
   }
   status = Worker::GetInstance().StartServable(servable, notify_master);

mindspore_serving/ccsrc/worker/distributed_worker/distributed_servable.cc

@@ -483,7 +483,7 @@ Status DistributedServable::WaitAgentsReady(uint64_t wait_agents_time_in_seconds
   uint64_t i;
   for (i = 0; i < kWaitMaxHundredMs; i++) {  //
     if (ExitSignalHandle::Instance().HasStopped()) {
-      return INFER_STATUS_LOG_ERROR(FAILED) << "Agents has stopped";
+      return INFER_STATUS_LOG_ERROR(FAILED) << "Worker or Agents has stopped";
     }
     // waiting for 100ms
     if (future.wait_for(std::chrono::milliseconds(100)) == std::future_status::ready) {
@@ -560,20 +560,25 @@ Status DistributedServable::CheckAgentsInfosAndInitTensorInfos() {
       return INFER_STATUS_LOG_ERROR(FAILED) << "Expect rank " << i << " input count equal to 0";
     }
   }
-  for (size_t i = 0; i < rank_size; i++) {
+  for (size_t i = 0; i < rank_size; i += parallel_count) {
     auto &first_item = agent_spec_map_[i];
     for (size_t k = 0; k < parallel_count && i + k < rank_size; k++) {
       auto rank_id = i + k;
       auto &agent_spec = agent_spec_map_[i + k];
       status = CompareTensorInfos(agent_spec.agent_spec_.output_infos, first_item.agent_spec_.output_infos);
       if (status != SUCCESS) {
-        status = INFER_STATUS_LOG_ERROR(FAILED) << "Rank " << rank_size << " output infos not match rank " << i
+        status = INFER_STATUS_LOG_ERROR(FAILED) << "Rank " << rank_id << " output infos not match rank " << i
                                                 << ", details: " << status.StatusMessage();
         return status;
       }
       if (agent_spec.agent_spec_.batch_size != 0 && agent_spec.agent_spec_.batch_size != batch_size_) {
+        if (!agent_spec.agent_spec_.output_infos.empty()) {
+          MSI_LOG_WARNING << "Rank " << rank_id << " output 0 shape: " << agent_spec.agent_spec_.output_infos[0].shape
+                          << ", batch size " << agent_spec.agent_spec_.batch_size;
+        }
         return INFER_STATUS_LOG_ERROR(FAILED)
-               << "Expect rank " << rank_id << " batch size equal to 0 or rank 0 batch size " << batch_size_;
+               << "Expect rank " << rank_id << " batch size  " << agent_spec.agent_spec_.batch_size
+               << " equal to 0 or rank 0's batch size " << batch_size_;
       }
     }
   }

third_party/mindspore

@@ -1 +1 @@
-Subproject commit e9a5d0248d12c323b1d0320321773d76c764a7bc
+Subproject commit 363e574ff8c7b6b7a695c05e0c34086c177e5d41