diff --git a/mindspore_serving/ccsrc/python/worker/worker_py.cc b/mindspore_serving/ccsrc/python/worker/worker_py.cc index 48b6464..fcd43df 100644 --- a/mindspore_serving/ccsrc/python/worker/worker_py.cc +++ b/mindspore_serving/ccsrc/python/worker/worker_py.cc @@ -46,6 +46,7 @@ void PyWorker::StartServable(const std::string &model_directory, const std::stri auto servable = std::make_shared(); auto status = servable->StartServable(model_directory, model_name, version_number); if (status != SUCCESS) { + servable->Clear(); MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage(); } status = Worker::GetInstance().StartServable(servable, notify_master); @@ -72,6 +73,7 @@ void PyWorker::StartServableInMaster(const std::string &model_directory, const s auto servable = std::make_shared(); auto status = servable->StartServable(model_directory, model_name, version_number); if (status != SUCCESS) { + servable->Clear(); MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage(); } status = Worker::GetInstance().StartServable(servable, notify_master); @@ -102,6 +104,7 @@ void PyWorker::StartDistributedServable(const std::string &servable_directory, c status = servable->StartServable(servable_directory, servable_name, rank_table_json_file, version_number, wait_agents_time_in_seconds); if (status != SUCCESS) { + servable->Clear(); MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage(); } status = Worker::GetInstance().StartServable(servable, notify_master); @@ -131,6 +134,7 @@ void PyWorker::StartDistributedServableInMaster(const std::string &servable_dire status = servable->StartServable(servable_directory, servable_name, rank_table_json_file, version_number, wait_agents_time_in_seconds); if (status != SUCCESS) { + servable->Clear(); MSI_LOG_EXCEPTION << "Raise failed: " << status.StatusMessage(); } status = Worker::GetInstance().StartServable(servable, notify_master); diff --git a/mindspore_serving/ccsrc/worker/distributed_worker/distributed_servable.cc b/mindspore_serving/ccsrc/worker/distributed_worker/distributed_servable.cc index 9cb043a..3e6f463 100644 --- a/mindspore_serving/ccsrc/worker/distributed_worker/distributed_servable.cc +++ b/mindspore_serving/ccsrc/worker/distributed_worker/distributed_servable.cc @@ -483,7 +483,7 @@ Status DistributedServable::WaitAgentsReady(uint64_t wait_agents_time_in_seconds uint64_t i; for (i = 0; i < kWaitMaxHundredMs; i++) { // if (ExitSignalHandle::Instance().HasStopped()) { - return INFER_STATUS_LOG_ERROR(FAILED) << "Agents has stopped"; + return INFER_STATUS_LOG_ERROR(FAILED) << "Worker or Agents has stopped"; } // waiting for 100ms if (future.wait_for(std::chrono::milliseconds(100)) == std::future_status::ready) { @@ -560,20 +560,25 @@ Status DistributedServable::CheckAgentsInfosAndInitTensorInfos() { return INFER_STATUS_LOG_ERROR(FAILED) << "Expect rank " << i << " input count equal to 0"; } } - for (size_t i = 0; i < rank_size; i++) { + for (size_t i = 0; i < rank_size; i += parallel_count) { auto &first_item = agent_spec_map_[i]; for (size_t k = 0; k < parallel_count && i + k < rank_size; k++) { auto rank_id = i + k; auto &agent_spec = agent_spec_map_[i + k]; status = CompareTensorInfos(agent_spec.agent_spec_.output_infos, first_item.agent_spec_.output_infos); if (status != SUCCESS) { - status = INFER_STATUS_LOG_ERROR(FAILED) << "Rank " << rank_size << " output infos not match rank " << i + status = INFER_STATUS_LOG_ERROR(FAILED) << "Rank " << rank_id << " output infos not match rank " << i << ", details: " << status.StatusMessage(); return status; } if (agent_spec.agent_spec_.batch_size != 0 && agent_spec.agent_spec_.batch_size != batch_size_) { + if (!agent_spec.agent_spec_.output_infos.empty()) { + MSI_LOG_WARNING << "Rank " << rank_id << " output 0 shape: " << agent_spec.agent_spec_.output_infos[0].shape + << ", batch size " << agent_spec.agent_spec_.batch_size; + } return INFER_STATUS_LOG_ERROR(FAILED) - << "Expect rank " << rank_id << " batch size equal to 0 or rank 0 batch size " << batch_size_; + << "Expect rank " << rank_id << " batch size " << agent_spec.agent_spec_.batch_size + << " equal to 0 or rank 0's batch size " << batch_size_; } } } diff --git a/third_party/mindspore b/third_party/mindspore index e9a5d02..363e574 160000 --- a/third_party/mindspore +++ b/third_party/mindspore @@ -1 +1 @@ -Subproject commit e9a5d0248d12c323b1d0320321773d76c764a7bc +Subproject commit 363e574ff8c7b6b7a695c05e0c34086c177e5d41