Huawei_Technology
/
MindSpore-GraphEngine
Not watched
1
0
Fork 0
Code
Releases 10 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

add error msg report

pull/1312/head
wangxiaotian22 5 years ago
parent
7fad65a58e
commit
5d83417155
4 changed files with 86 additions and 10 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +2
    -2
      ge/graph/build/model_builder.cc
  2. +2
    -2
      ge/graph/build/stream_allocator.cc
  3. +5
    -2
      ge/graph/build/stream_graph_optimizer.cc
  4. +77
    -4
      ge/graph/build/task_generator.cc

+ 2
- 2
ge/graph/build/model_builder.cc View File

@@ -848,8 +848,8 @@ Status ModelBuilder::CompileSingleOp() {
if (kernel_info != nullptr) {
node_vector_map[kernel_lib_name].emplace_back(node);
} else {
REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s) when %s",
node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__);
REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s, when %s",
node->GetName().c_str(), node->GetType().c_str(), kernel_lib_name.c_str(), __FUNCTION__);
GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s ops kernel info store failed", node->GetName().c_str());
return ge::GE_GRAPH_PARAM_NULLPTR;
}


+ 2
- 2
ge/graph/build/stream_allocator.cc View File

@@ -1470,7 +1470,7 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector<string
for (size_t i = 0; i < label_num; i++) {
const string &active_label = ori_active_label_list[i];
if (labeled_streams_.find(active_label) == labeled_streams_.end()) {
REPORT_INNER_ERROR("can not find stream label:%s when %s", active_label.c_str(), __FUNCTION__);
REPORT_INNER_ERROR("E19999", "can not find stream label:%s when %s", active_label.c_str(), __FUNCTION__);
GELOGE(FAILED, "can not find stream label %s", active_label.c_str());
return FAILED;
}
@@ -1536,7 +1536,7 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector<string

Status StreamAllocator::SetActiveStreamList(NodePtr &active_node, const string &active_label) {
if (labeled_streams_.find(active_label) == labeled_streams_.end()) {
REPORT_INNER_ERROR("Can not find stream label:%s when %s", active_label.c_str(), __FUNCTION__);
REPORT_INNER_ERROR("E19999", "Can not find stream label:%s when %s", active_label.c_str(), __FUNCTION__);
GELOGE(FAILED, "Can not find stream label %s.", active_label.c_str());
return FAILED;
}


+ 5
- 2
ge/graph/build/stream_graph_optimizer.cc View File

@@ -14,6 +14,9 @@
* limitations under the License.
*/
#include "stream_graph_optimizer.h"

#include <securec.h>

#include "common/util.h"
#include "framework/common/debug/ge_log.h"
#include "graph/utils/node_utils.h"
@@ -122,7 +125,7 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com
GE_CHECK_NOTNULL(op_desc);
int64_t stream_id = op_desc->GetStreamId();
if (static_cast<size_t>(stream_id) >= run_context.graphStreamList.size()) {
REPORT_INNER_ERROR("Check stream_id:%ld in op:%s(%s) is bigger than run_context.graphStreamList.size():%zu "
REPORT_INNER_ERROR("E19999", "Check stream_id:%ld in op:%s(%s) is bigger than run_context.graphStreamList.size():%zu "
"when %s", stream_id, op_desc->GetName().c_str(),
op_desc->GetType().c_str(), run_context.graphStreamList.size(), __FUNCTION__);
GELOGE(FAILED, "stream_id %ld is bigger than run_context.graphStreamList.size() %zu", stream_id,
@@ -138,7 +141,7 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com
for (auto iter = graph_optimizers.begin(); iter != graph_optimizers.end(); ++iter) {
GE_CHECK_NOTNULL(*iter);
Status ret = (*iter)->OptimizeStreamGraph(*subgraph, run_context);
REPORT_CALL_ERROR("Call optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph "
REPORT_CALL_ERROR("E19999", "Call optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph "
"Optimizer num: %zu, ret: %u", subgraph->GetName().c_str(), engine_name.c_str(),
graph_optimizers.size(), ret);
if (ret != SUCCESS) {


+ 77
- 4
ge/graph/build/task_generator.cc View File

@@ -69,6 +69,7 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t
GELOGD("Begin to Get TaskInfo. session_id=%lu", session_id);
// Check params
if (graph == nullptr) {
REPORT_INNER_ERROR("E19999", "Check param graph is null, session_id:%lu, when %s", session_id, __FUNCTION__);
GELOGE(PARAM_INVALID, "GetTaskInfo param graph is null. session_id=%lu", session_id);
return PARAM_INVALID;
}
@@ -93,6 +94,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t
op_name.push_back(iter.second);
}
GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(model, ATTR_MODEL_TASK_INDEX_OP_NAME, op_name),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s",
ATTR_MODEL_TASK_INDEX_OP_NAME.c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetListStr failed.");
return FAILED);

@@ -106,6 +109,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t
for (const TaskDef &task_def_temp : task_def_list) {
TaskDef *task_def = model_task_def.add_task();
if (task_def == nullptr) {
REPORT_INNER_ERROR("E19999", "Add task_def in ModelTaskDef fail, session_id:%lu, graph:%s, model:%s, when %s",
session_id, graph->GetName().c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "task_def is nullptr.");
return FAILED;
}
@@ -126,30 +131,44 @@ Status TaskGenerator::AddModelTaskToModel(const ModelTaskDef &model_task_def, ui
RunContext &run_context) {
GE_CHK_BOOL_EXEC(
AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_BASE_ADDR, reinterpret_cast<uintptr_t>(run_context.dataMemBase)),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s",
MODEL_ATTR_TASK_GEN_BASE_ADDR.c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_BASE_ADDR failed.");
return FAILED);
GE_CHK_BOOL_EXEC(
AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_WEIGHT_ADDR, reinterpret_cast<uintptr_t>(run_context.weightMemBase)),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s",
MODEL_ATTR_TASK_GEN_WEIGHT_ADDR.c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_WEIGHT_ADDR failed.");
return FAILED);
GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_TASK_GEN_VAR_ADDR, reinterpret_cast<uintptr_t>(var_mem_base_)),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s",
ATTR_MODEL_TASK_GEN_VAR_ADDR.c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetInt ATTR_MODEL_TASK_GEN_VAR_ADDR failed.");
return FAILED);
GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_VAR_SIZE, var_mem_size_),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s",
ATTR_MODEL_VAR_SIZE.c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetInt ATTR_MODEL_VAR_SIZE failed.");
return FAILED);
GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, MODEL_ATTR_SESSION_ID, session_id),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for mode:%s when %s",
MODEL_ATTR_SESSION_ID.c_str(), model.GetName().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetInt MODEL_ATTR_SESSION_ID failed.");
return FAILED);

size_t task_size = model_task_def.ByteSizeLong();
ge::Buffer serial_buff(task_size);
if (!model_task_def.SerializePartialToArray(serial_buff.GetData(), static_cast<int>(task_size))) {
REPORT_INNER_ERROR("E19999", "model_task_def's serialize failed, model name = %s, task_size=%zu when %s",
model.GetName().c_str(), task_size, __FUNCTION__);
GELOGE(FAILED, "model_task_def's serialize failed, model name = %s, task_size=%zu.", model.GetName().c_str(),
task_size);
return FAILED;
}
if (!AttrUtils::SetZeroCopyBytes(model, MODEL_ATTR_TASKS, std::move(serial_buff))) {
REPORT_INNER_ERROR("E19999", "Set model task to model failed, model name = %s, task_size=%zu.",
model.GetName().c_str(), task_size, __FUNCTION__);
GELOGE(FAILED, "Set model task to model failed, model name = %s, task_size=%zu.", model.GetName().c_str(),
task_size);
return FAILED;
@@ -167,7 +186,10 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi
for (int64_t input : input_offsets) {
input_var.push_back(VarManager::Instance(session_id)->IsVarAddr(input));
}
GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var), GELOGE(FAILED, "SetListBool failed.");
GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsInputVar,
op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetListBool failed.");
return FAILED);
}

@@ -177,7 +199,10 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi
for (int64_t output : output_offsets) {
output_var.push_back(VarManager::Instance(session_id)->IsVarAddr(output));
}
GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var), GELOGE(FAILED, "SetListBool failed.");
GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsOutputVar,
op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__);
GELOGE(FAILED, "SetListBool failed.");
return FAILED);
}
return SUCCESS;
@@ -252,6 +277,7 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
GELOGD("Beign to generate task, graph name is %s.", graph->GetName().c_str());
std::shared_ptr<GELib> ge_lib = GELib::GetInstance();
if ((ge_lib == nullptr) || !ge_lib->InitFlag()) {
REPORT_INNER_ERROR("E19999", "Check GELib instance not init before when %s", __FUNCTION__);
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GenerateTask failed.");
return GE_CLI_GE_NOT_INITIALIZED;
}
@@ -319,6 +345,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
}
auto kernel_info_store = ops_kernel_manager.GetOpsKernelInfoStore(op_kernel_lib_name);
if (kernel_info_store == nullptr) {
REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s when %s",
node->GetName().c_str(), node->GetType().c_str(), op_kernel_lib_name.c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR,
"No ops kernel store or ops kernel builder found. node:%s(%s), op_kernel_lib_name=%s.",
name.c_str(),
@@ -344,6 +372,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
auto ret = OpsKernelBuilderManager::Instance().GenerateTask(*node, run_context, task_def_list);
GE_TIMESTAMP_ADD(GenerateTask);
if (ret != SUCCESS) {
REPORT_CALL_ERROR("E19999", "Call OpsKernelBuilderManager GenerateTask fail for op:%s(%s) when %s",
node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__);
GELOGE(ret, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task failed.",
op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id);
return ret;
@@ -353,6 +383,9 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
size_t task_list_size_after = task_def_list.size();
// If tasks is reduced
if (task_list_size_after < task_list_size_before) {
REPORT_INNER_ERROR("E19999", "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task "
"but task num from %zu to %zu, check invalid", op_kernel_lib_name.c_str(), name.c_str(),
type.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after);
GELOGE(FAILED, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task. but task num from %zu to %zu.",
op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id, task_list_size_before,
task_list_size_after);
@@ -417,6 +450,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
size_t task_list_size_before = task_def_list.size();
OpsKernelInfoStorePtr kernel_info_store = ops_kernel_manager.GetOpsKernelInfoStore(op_kernel_lib_name);
if (kernel_info_store == nullptr) {
REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s when %s",
op_desc->GetName().c_str(), op_desc->GetType().c_str(),
op_kernel_lib_name.c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR,
"Fusion: No ops kernel store or ops kernel builder found. fusion_node:%s(%s), op_kernel_lib_name=%s.",
fusion_node_name.c_str(), fusion_node_type.c_str(), op_kernel_lib_name.c_str());
@@ -433,6 +469,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
int64_t op_id = op_desc->GetId();
int64_t stream_id = op_desc->GetStreamId();
if (stream_id < 0 || stream_id >= (int64_t)run_context.graphStreamList.size()) {
REPORT_INNER_ERROR("E19999", "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, "
"stream list size=%zu, when %s", fusion_node_name.c_str(), fusion_node_type.c_str(),
op_id, run_context.graphStreamList.size(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, stream list size=%zu",
fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, run_context.graphStreamList.size());
return INTERNAL_ERROR;
@@ -444,6 +483,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id);
ret = OpsKernelBuilderManager::Instance().GenerateTask(*fusion_node, run_context, task_def_list);
if (ret != SUCCESS) {
REPORT_CALL_ERROR("E19999", " Call %s to generate fusion_node:[fusion_node_name:%s(%s), "
"id:%ld, stream_id:%ld] task failed when %s", op_kernel_lib_name.c_str(),
fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id, __FUNCTION__);
GELOGE(ret,
"Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), "
"id:%ld, stream_id:%ld] task failed.",
@@ -455,6 +497,10 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
size_t task_list_size_after = task_def_list.size();
// if tasks is reduced
if (task_list_size_after < task_list_size_before) {
REPORT_INNER_ERROR("E19999", "InsertProfilingTask for fusion_node:[fusion_node_name:%s(%s), "
"id:%ld, stream_id:%ld] task, but task num from %zu to %zu, check invalid when %s",
op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(),
op_id, stream_id, task_list_size_before, task_list_size_after, __FUNCTION__);
GELOGE(FAILED,
"Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), "
"id:%ld, stream_id:%ld] task. but task num from %zu to %zu.",
@@ -489,6 +535,8 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info

Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
if (NodeUtils::SetAllAnchorStatus(node) != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "SetAllAnchorStatus fail for op:%s(%s) when %s",
node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "NodeUtils::SetAllAnchorStatus failed.");
return INTERNAL_ERROR;
}
@@ -496,6 +544,8 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
auto peer_anchor = anchor->GetPeerOutAnchor();
if (peer_anchor == nullptr) {
if (AnchorUtils::SetStatus(anchor, ANCHOR_SUSPEND) != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Set in peer anchor status fail for op:%s(%s), anchor_index:%d, when %s",
node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed.");
return INTERNAL_ERROR;
}
@@ -506,11 +556,15 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
bool is_const = NodeUtils::GetConstOpType(peer_anchor->GetOwnerNode(), const_type);
if (is_const && (const_type == CONSTANT)) {
if (AnchorUtils::SetStatus(anchor, ANCHOR_CONST) != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Set in anchor CONST status fail for op:%s(%s), anchor_index:%d, when %s",
node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed.");
return INTERNAL_ERROR;
}
} else {
if (AnchorUtils::SetStatus(anchor, ANCHOR_DATA) != GRAPH_SUCCESS) {
REPORT_CALL_ERROR("E19999", "Set in anchor DATA status fail for op:%s(%s), anchor_index:%d, when %s",
node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed.");
return INTERNAL_ERROR;
}
@@ -523,12 +577,15 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
Status TaskGenerator::MarkNodeAndSetIndex(ComputeGraphPtr &graph) {
auto ge_lib = GELib::GetInstance();
if ((ge_lib == nullptr) || !ge_lib->InitFlag()) {
REPORT_INNER_ERROR("E19999", "Check GELib instance not init before when %s", __FUNCTION__);
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized or is finalized.");
return GE_CLI_GE_NOT_INITIALIZED;
}

const auto all_nodes = graph->GetNodes(graph->GetGraphUnknownFlag());
if (all_nodes.empty()) {
REPORT_INNER_ERROR("E19999", "Check param all_nodes empty in graph:%s when %s",
graph->GetName().c_str(), __FUNCTION__);
GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Graph's node is empty");
return GE_GRAPH_GRAPH_NODE_NULL;
}
@@ -584,6 +641,9 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector<OpDescPtr> &ops, bool is_
for (auto &op_desc : continuous_ops) {
string op_kernel_lib_name = op_desc->GetOpKernelLibName();
if (op_kernel_lib_name.empty()) {
REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s when %s",
op_desc->GetName().c_str(), op_desc->GetType().c_str(),
op_kernel_lib_name.c_str(), __FUNCTION__);
GELOGE(INTERNAL_ERROR, "node:%s(%s) get op kernel lib failed.", op_desc->GetName().c_str(),
op_desc->GetType().c_str());
return INTERNAL_ERROR;
@@ -599,9 +659,17 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector<OpDescPtr> &ops, bool is_

for (auto &it : first_and_last_ops) {
auto &op_pair = it.second;
GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true), GELOGE(FAILED, "SetBool failed.");
GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsFirstNode,
op_pair.first->GetName().c_str(), op_pair.first->GetType().c_str(),
__FUNCTION__);
GELOGE(FAILED, "SetBool failed.");
return FAILED);
GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true), GELOGE(FAILED, "SetBool failed.");
GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true),
REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsLastNode,
op_pair.second->GetName().c_str(), op_pair.second->GetType().c_str(),
__FUNCTION__);
GELOGE(FAILED, "SetBool failed.");
return FAILED);
}
}
@@ -906,6 +974,8 @@ Status TaskGenerator::InsertProfilingArTaskBefore(const OpDescPtr &op_desc, std:
for (size_t i = 0; i < all_reduce_nodes.size(); i++) {
if (all_reduce_nodes[i] == node_index) {
GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep),
REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id "
"for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str());
GELOGE(FAILED, "Multiply result is out of range.");
return FAILED);
ar_log_id = i * kProfilingArStep + kProfilingArStartLogid;
@@ -998,6 +1068,8 @@ Status TaskGenerator::InsertProfilingArTaskAfter(const OpDescPtr &op_desc, std::
for (size_t i = 0; i < all_reduce_nodes.size(); i++) {
if (all_reduce_nodes[i] == node_index) {
GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep),
REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id "
"for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str());
GELOGE(FAILED, "Multiply result is out of range.");
return FAILED);
ar_log_id = i * kProfilingArStep + kProfilingArEndLogid;
@@ -1107,6 +1179,7 @@ Status TaskGenerator::SetUnknownShapeStream(RunContext &run_context, rtStream_t
run_context.stream = stream;
rtError_t rt_ret = rtModelBindStream(run_context.model, stream, 0);
if (rt_ret != RT_ERROR_NONE) {
REPORT_CALL_ERROR("E19999", "Call rtModelBindStream fail, ret:0x%X when %s", ret, __FUNCTION__);
GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
GE_CHK_RT_RET(rtStreamDestroy(stream));
return FAILED;


Write Preview
Loading…
Cancel
Save